From 06018439f9c39b3a83b04db8e7b74061db6b8f56 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 01:21:56 +0100 Subject: [PATCH 001/176] Aggiunto controllo build versione. --- checkbuilt.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 checkbuilt.sh diff --git a/checkbuilt.sh b/checkbuilt.sh new file mode 100644 index 0000000..e86d165 --- /dev/null +++ b/checkbuilt.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +storage=$1 +version=$2 +shift 2 + +count=$(find "${storage}" -iname "*${version}*.AppImage" | wc -l) + +if [[ ${count} -eq 0 ]]; then + echo "status: built" +else + echo "status: not built" From 59ff822ca19be47c55b7995da9cf160b6b41e74c Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 01:27:48 +0100 Subject: [PATCH 002/176] Correttoscript di verifica build. --- checkbuilt.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/checkbuilt.sh b/checkbuilt.sh index e86d165..19c48bd 100644 --- a/checkbuilt.sh +++ b/checkbuilt.sh @@ -10,3 +10,4 @@ if [[ ${count} -eq 0 ]]; then echo "status: built" else echo "status: not built" +fi From c7648c520ba436b691461e3e66258eb64d91b06d Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 03:34:53 +0100 Subject: [PATCH 003/176] Corretto script check build. Costruzione di uno script di build. --- build.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++ checkbuilt.sh | 6 ++-- 2 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 build.py diff --git a/build.py b/build.py new file mode 100644 index 0000000..d94e3ab --- /dev/null +++ b/build.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +import urllib.request +from lxml import etree +import tempfile, os, subprocess + +class Build: + LANGSTD = [ 'en-GB', 'it', 'ar', 'zh-CN', 'zh-TW', 'fr', 'de', 'ja', 'ko', 'pt', 'pt-BR', 'es', 'ru' ] + + def __init__(self, query, arch, url): + """Build all versions that can be found in the indicated repo.""" + self.__query__ = query + self.__arch__ = arch + self.__url__ = url + + # Creating a tempfile + self.__builddir__ = tempfile.mktempdir() + self.__tarballs__ = [] + self.__appname__ = '' + self.__version__ = '' + + if self.__url__ == '-': + print("Cannot find this version for arch {arch}.".format(arch = self.__arch__)) + return False + + + def download(self): + """Downloads the contents of the URL as it was a folder.""" + + # Let's start with defining which files are to be downloaded. + # Let's explore the remote folder. + contents = etree.HTML(urllib.request.openurl(self.__url__).read()).xpath("//td/a") + self.__tarballs__ = [ x.text for x in contents if x.text.endswith('tar.gz') ] + maintarball = self.__tarballs__[0] + main_arr = maintarball.split('_') + self.__appname__ = main_arr[0] + self.__version__ = main_arr[1] + + os.chdir(self.__builddir__) + for archive in self.__tarballs__: + # Download the archive + try: + urllib.request.urlretrieve(self.__url__ + archive, archive) + except: + print("Failed to download {archive}.".format(archive = archive)) + + print("Got %s." % archive) + + + def build(self, langset = 'full') + """Building all the versions.""" + # We have 4 builds to do: + # * standard languages, no help + # * standard languages + offline help + # * all languages, no help + # * all languages + offline help + + # Let's start with the quickest build: standard languages, no help. + # We start by filtering out tarballs from the list + buildtarballs = [ self.__tarballs__[0] ] + + # Let's process standard languages and append results to the + # buildtarball + for lang in LANGSTD: + buildtarballs.extend([ x for x in self.__tarballs__ if ('langpack_' + lang) in x ]) + # If it was a build with offline help, another extend would have + # solved it: + #buildtarballs.extend([ x for x in self.__tarballs__ if ('helppack_'+ lang) in x ]) + + # Creating a subfolder + os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage'), exist_ok = True) + # And then cd to the appname folder. + os.chdir(os.path.join(self.__builddir__, self.__appname__)) + + # Unpacking the tarballs + for archive in buildtarballs: + subprocess.call("tar xzf ../%s.tar.gz" % archive, shell=True) + + # At this point, let's decompress the deb packages + os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) + subprocess.call("find .. -iname '*.deb' -exec dpkg -x {} . \+", shell=True) + # Changing desktop file + subprocess.call("find . -iname startcenter.desktop -exec cp {} . \+", shell=True) + subprocess.call("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) + subprocess.call("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} \;", shell=True) + + binaryname = subprocess.check_output("cat startcenter.desktop | awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' | awk '{ print $1 }'", shell=True).strip('\n') + subprocess.call("rm -f usr/bin/%s" % binaryname, shell=True) + subprocess.call("find opt -name soffice -path '*programm*' -exec ln -s {} usr/bin/%si \;" % binaryname, shell=True) + # Arrivato a riga 501 + + + + + + + diff --git a/checkbuilt.sh b/checkbuilt.sh index 19c48bd..37ac7e0 100644 --- a/checkbuilt.sh +++ b/checkbuilt.sh @@ -7,7 +7,9 @@ shift 2 count=$(find "${storage}" -iname "*${version}*.AppImage" | wc -l) if [[ ${count} -eq 0 ]]; then - echo "status: built" -else echo "status: not built" + exit 1 +else + echo "status: built" + # Nothing more to do! fi From fb2f9fac4525e9d7199cf54c88727bd4d202f953 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:08:20 +0100 Subject: [PATCH 004/176] Prima versione finale di build. --- build.py | 66 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/build.py b/build.py index d94e3ab..8e7fac2 100644 --- a/build.py +++ b/build.py @@ -2,10 +2,10 @@ import urllib.request from lxml import etree -import tempfile, os, subprocess +import tempfile, os, sys, subprocess, shutils -class Build: - LANGSTD = [ 'en-GB', 'it', 'ar', 'zh-CN', 'zh-TW', 'fr', 'de', 'ja', 'ko', 'pt', 'pt-BR', 'es', 'ru' ] +class Build(): + LANGSTD = ['ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW'] def __init__(self, query, arch, url): """Build all versions that can be found in the indicated repo.""" @@ -47,7 +47,7 @@ class Build: print("Got %s." % archive) - def build(self, langset = 'full') + def build(self, outdir): """Building all the versions.""" # We have 4 builds to do: # * standard languages, no help @@ -74,24 +74,60 @@ class Build: # Unpacking the tarballs for archive in buildtarballs: - subprocess.call("tar xzf ../%s.tar.gz" % archive, shell=True) + subprocess.run("tar xzf ../%s.tar.gz" % archive, shell=True) # At this point, let's decompress the deb packages os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) - subprocess.call("find .. -iname '*.deb' -exec dpkg -x {} . \+", shell=True) + subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \+", shell=True) # Changing desktop file - subprocess.call("find . -iname startcenter.desktop -exec cp {} . \+", shell=True) - subprocess.call("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) - subprocess.call("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} \;", shell=True) - - binaryname = subprocess.check_output("cat startcenter.desktop | awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' | awk '{ print $1 }'", shell=True).strip('\n') - subprocess.call("rm -f usr/bin/%s" % binaryname, shell=True) - subprocess.call("find opt -name soffice -path '*programm*' -exec ln -s {} usr/bin/%si \;" % binaryname, shell=True) - # Arrivato a riga 501 - + subprocess.run("find . -iname startcenter.desktop -exec cp {} . \+", shell=True) + subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) + subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} \;", shell=True) + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).strip('\n') + subprocess.run("rm -f usr/bin/%s" % binaryname, shell=True) + os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage', 'usr', 'bin')) + subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %si \;" % binaryname, shell=True) + os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) + + # Download AppRun from github + apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) + urllib.request.urlretrieve(apprunurl, 'AppRun') + os.chmod('AppRun', 0o755) + + os.chdir(os.path.join(self.__builddir__, self.__appname__)) + # Download appimagetool from probonopd repo on github + appimagetoolurl = "https://github.com/probonopd/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) + urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') + os.chmod('appimagetool', 0o755) + + # Setting app version + appversion = self.__version__ + '.standard' + # Building app + subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) + + # Copying built image to final directory + subprocess.run("find . -iname '*.AppImage' -exec cp {} %s \;" % outdir, shell = True) + + def __del__(self): + """Destructor for the class.""" + # Cleanup + shutils.rmtree(self.__builddir__) +if __name__ == '__main__': + # Run if it is run as a program. + # 1 -> query + # 2 -> arch + # 3 -> url + # 4 -> outdir + if not len(sys.argv) == 5: + print("Please launch with this parameters: build.py query arch url outputdir") + sys.exit(255) + b = Build(sys.argv[1], sys.argv[2], sys.argv[3]) + b.download() + b.build("/var/tmp/rundeck/AppImageStorage/") + del b From 1bc882508def97a8f4bbae82d5fbdc0ba4ca3448 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:09:11 +0100 Subject: [PATCH 005/176] Correzione chiamata con parametri. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 8e7fac2..0ab721a 100644 --- a/build.py +++ b/build.py @@ -128,6 +128,6 @@ if __name__ == '__main__': b = Build(sys.argv[1], sys.argv[2], sys.argv[3]) b.download() - b.build("/var/tmp/rundeck/AppImageStorage/") + b.build(sys.argv[4]) del b From e2958932b8183f4a19642a6650dba82a310ba875 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:15:59 +0100 Subject: [PATCH 006/176] Correzione libreria. --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 0ab721a..ac7742e 100644 --- a/build.py +++ b/build.py @@ -2,7 +2,7 @@ import urllib.request from lxml import etree -import tempfile, os, sys, subprocess, shutils +import tempfile, os, sys, subprocess, shutil class Build(): LANGSTD = ['ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW'] @@ -112,7 +112,7 @@ class Build(): def __del__(self): """Destructor for the class.""" # Cleanup - shutils.rmtree(self.__builddir__) + shutil.rmtree(self.__builddir__) if __name__ == '__main__': From c1f3431fc1cf650e896573cbb0dbbbc37cf91216 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:20:59 +0100 Subject: [PATCH 007/176] Correzione richiamo directory. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index ac7742e..be4bced 100644 --- a/build.py +++ b/build.py @@ -14,7 +14,7 @@ class Build(): self.__url__ = url # Creating a tempfile - self.__builddir__ = tempfile.mktempdir() + self.__builddir__ = tempfile.mkdtemp() self.__tarballs__ = [] self.__appname__ = '' self.__version__ = '' From 462bce15adad5e57e80e314a4795d60269d6bcfe Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:23:11 +0100 Subject: [PATCH 008/176] Corretto urlopen. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index be4bced..23a1a69 100644 --- a/build.py +++ b/build.py @@ -29,7 +29,7 @@ class Build(): # Let's start with defining which files are to be downloaded. # Let's explore the remote folder. - contents = etree.HTML(urllib.request.openurl(self.__url__).read()).xpath("//td/a") + contents = etree.HTML(urllib.request.urlopen(self.__url__).read()).xpath("//td/a") self.__tarballs__ = [ x.text for x in contents if x.text.endswith('tar.gz') ] maintarball = self.__tarballs__[0] main_arr = maintarball.split('_') From 4f9aae424cd09fd804d0f10404187ec90625d69f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:30:52 +0100 Subject: [PATCH 009/176] Cambio output per variabili. --- getversions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/getversions.py b/getversions.py index cf09a20..e4664a5 100644 --- a/getversions.py +++ b/getversions.py @@ -81,8 +81,8 @@ if __name__ == '__main__': version = a['version'] basedirurl = a['basedirurl'] - output = """query: %s -version: %s -x86: %s -x86_64: %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) + output = """query = %s +version = %s +x86 = %s +x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) print(output) From 18a53ad0f4fa8ec2a464be28c4e176674113d6e7 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:33:56 +0100 Subject: [PATCH 010/176] Cambio output per variabili per Rundeck. --- getversions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/getversions.py b/getversions.py index e4664a5..af6ca7e 100644 --- a/getversions.py +++ b/getversions.py @@ -81,8 +81,8 @@ if __name__ == '__main__': version = a['version'] basedirurl = a['basedirurl'] - output = """query = %s -version = %s -x86 = %s -x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) + output = """RUNDECK.DATA: query = %s +RUNDECK.DATA: version = %s +RUNDECK.DATA: x86 = %s +RUNDECK.DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) print(output) From 93f7d95e3db47b48b83d3d5cb261d3a9981686ce Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:35:40 +0100 Subject: [PATCH 011/176] Cambio output per variabili per Rundeck -3. --- getversions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/getversions.py b/getversions.py index af6ca7e..91e1d4d 100644 --- a/getversions.py +++ b/getversions.py @@ -82,7 +82,7 @@ if __name__ == '__main__': basedirurl = a['basedirurl'] output = """RUNDECK.DATA: query = %s -RUNDECK.DATA: version = %s -RUNDECK.DATA: x86 = %s -RUNDECK.DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) +RUNDECK:DATA: version = %s +RUNDECK:DATA: x86 = %s +RUNDECK:DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) print(output) From 2f1591cb5dba42b9723862a66a1e74c121402ea3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:37:04 +0100 Subject: [PATCH 012/176] Cambio output per variabili per Rundeck - 4. --- getversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getversions.py b/getversions.py index 91e1d4d..b6acdd8 100644 --- a/getversions.py +++ b/getversions.py @@ -81,7 +81,7 @@ if __name__ == '__main__': version = a['version'] basedirurl = a['basedirurl'] - output = """RUNDECK.DATA: query = %s + output = """RUNDECK:DATA: query = %s RUNDECK:DATA: version = %s RUNDECK:DATA: x86 = %s RUNDECK:DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) From d4d279991a8cde3b73b5f6e187ed948995f2681b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:48:54 +0100 Subject: [PATCH 013/176] Correzione allo script di build. --- build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 23a1a69..5d37b80 100644 --- a/build.py +++ b/build.py @@ -4,8 +4,8 @@ import urllib.request from lxml import etree import tempfile, os, sys, subprocess, shutil -class Build(): - LANGSTD = ['ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW'] +class Build(object): + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] def __init__(self, query, arch, url): """Build all versions that can be found in the indicated repo.""" @@ -61,7 +61,7 @@ class Build(): # Let's process standard languages and append results to the # buildtarball - for lang in LANGSTD: + for lang in Build.LANGSTD: buildtarballs.extend([ x for x in self.__tarballs__ if ('langpack_' + lang) in x ]) # If it was a build with offline help, another extend would have # solved it: From 18f06aedff7120ccad17ba5b836045d6765f198d Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 22:53:52 +0100 Subject: [PATCH 014/176] Corretto unpacking dei tarball. --- build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 5d37b80..f0925c8 100644 --- a/build.py +++ b/build.py @@ -74,7 +74,7 @@ class Build(object): # Unpacking the tarballs for archive in buildtarballs: - subprocess.run("tar xzf ../%s.tar.gz" % archive, shell=True) + subprocess.run("tar xzf ../%s" % archive, shell=True) # At this point, let's decompress the deb packages os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) @@ -108,11 +108,12 @@ class Build(object): # Copying built image to final directory subprocess.run("find . -iname '*.AppImage' -exec cp {} %s \;" % outdir, shell = True) + os.chdir('/tmp') def __del__(self): """Destructor for the class.""" # Cleanup - shutil.rmtree(self.__builddir__) + del self.__builddir__ if __name__ == '__main__': From 29984163c328acf7c8f24cb103a4138b117ab390 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 23:01:14 +0100 Subject: [PATCH 015/176] Correzioni ai find lanciati da interno python. --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index f0925c8..6293434 100644 --- a/build.py +++ b/build.py @@ -78,9 +78,9 @@ class Build(object): # At this point, let's decompress the deb packages os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) - subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \+", shell=True) + subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file - subprocess.run("find . -iname startcenter.desktop -exec cp {} . \+", shell=True) + subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} \;", shell=True) From ef7202a7c65d487baf38f8edca6c664c67c74314 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 23:15:31 +0100 Subject: [PATCH 016/176] Correzione percorsi, creazione altre cartelle necessarie. --- build.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/build.py b/build.py index 6293434..be14453 100644 --- a/build.py +++ b/build.py @@ -68,7 +68,7 @@ class Build(object): #buildtarballs.extend([ x for x in self.__tarballs__ if ('helppack_'+ lang) in x ]) # Creating a subfolder - os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage'), exist_ok = True) + os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir'), exist_ok = True) # And then cd to the appname folder. os.chdir(os.path.join(self.__builddir__, self.__appname__)) @@ -77,18 +77,19 @@ class Build(object): subprocess.run("tar xzf ../%s" % archive, shell=True) # At this point, let's decompress the deb packages - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) + os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} \;", shell=True) - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).strip('\n') - subprocess.run("rm -f usr/bin/%s" % binaryname, shell=True) - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage', 'usr', 'bin')) + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') + os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) + + os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %si \;" % binaryname, shell=True) - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppImage')) + os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) # Download AppRun from github apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) @@ -109,11 +110,7 @@ class Build(object): # Copying built image to final directory subprocess.run("find . -iname '*.AppImage' -exec cp {} %s \;" % outdir, shell = True) os.chdir('/tmp') - - def __del__(self): - """Destructor for the class.""" - # Cleanup - del self.__builddir__ + del self.__builddir__ if __name__ == '__main__': From 6a08d3bcb2f55dc259c08235c6343230ba982cc7 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 23:18:57 +0100 Subject: [PATCH 017/176] Ancora correzioni sulla build. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index be14453..4dc6f82 100644 --- a/build.py +++ b/build.py @@ -88,7 +88,7 @@ class Build(object): os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) - subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %si \;" % binaryname, shell=True) + subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) # Download AppRun from github From 8649b928bbaff7d2798e57148b6062269ea9074b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 23:35:37 +0100 Subject: [PATCH 018/176] Revisione dell'icona. --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 4dc6f82..04db36a 100644 --- a/build.py +++ b/build.py @@ -82,11 +82,11 @@ class Build(object): # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) - subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} \;", shell=True) binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') + subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} %s-startcenter.png \;" % binaryname, shell=True) + os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) From 565c2d9dce3e08e9cea0f1dbe9b319829877912f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 23:38:26 +0100 Subject: [PATCH 019/176] Correzione creazione sottocartelle. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 04db36a..1382c92 100644 --- a/build.py +++ b/build.py @@ -86,7 +86,7 @@ class Build(object): binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} %s-startcenter.png \;" % binaryname, shell=True) - os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) + os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin'), exist_ok = True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) From 52860c8a62dfa4b827ba0204c26cda0d2070a6cd Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 19 Mar 2022 23:40:42 +0100 Subject: [PATCH 020/176] Ricreato distruttore. --- build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 1382c92..2d0c2fe 100644 --- a/build.py +++ b/build.py @@ -110,6 +110,8 @@ class Build(object): # Copying built image to final directory subprocess.run("find . -iname '*.AppImage' -exec cp {} %s \;" % outdir, shell = True) os.chdir('/tmp') + + def __del__(self): del self.__builddir__ @@ -127,5 +129,4 @@ if __name__ == '__main__': b = Build(sys.argv[1], sys.argv[2], sys.argv[3]) b.download() b.build(sys.argv[4]) - del b From 7a25fb5064e4ca42207ec3b3ecc648f61751abc8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 00:01:15 +0100 Subject: [PATCH 021/176] Correzione build. --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 2d0c2fe..878c277 100644 --- a/build.py +++ b/build.py @@ -81,10 +81,10 @@ class Build(object): subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) - subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) + #subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') - subprocess.run("find . -name startcenter.png -path '*hicolor*48x48*' -exec cp {} %s-startcenter.png \;" % binaryname, shell=True) + subprocess.run("find . -name \*startcenter.png -path '*hicolor*48x48*' -exec cp {} . \;" % binaryname, shell=True) os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin'), exist_ok = True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) From a6fdcdc2ebe0aff0cdb5b1cedce457388c3c9c5f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 00:08:03 +0100 Subject: [PATCH 022/176] Ancora sistemazione icona. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 878c277..abb5956 100644 --- a/build.py +++ b/build.py @@ -84,7 +84,7 @@ class Build(object): #subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') - subprocess.run("find . -name \*startcenter.png -path '*hicolor*48x48*' -exec cp {} . \;" % binaryname, shell=True) + subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;" % binaryname, shell=True) os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin'), exist_ok = True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) From e837dd600ef84d95ca9cd5ae1e4581127b0cbb9d Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 00:11:09 +0100 Subject: [PATCH 023/176] Rimozione string replacement non necessario. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index abb5956..64f9970 100644 --- a/build.py +++ b/build.py @@ -84,7 +84,7 @@ class Build(object): #subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') - subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;" % binaryname, shell=True) + subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin'), exist_ok = True) os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) From e51cf40b767d189a3ab567f9a903ba5a79216066 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 00:24:35 +0100 Subject: [PATCH 024/176] Non faccio rimuovere la cartella per ulteriori controlli. --- build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 64f9970..741c29f 100644 --- a/build.py +++ b/build.py @@ -97,8 +97,8 @@ class Build(object): os.chmod('AppRun', 0o755) os.chdir(os.path.join(self.__builddir__, self.__appname__)) - # Download appimagetool from probonopd repo on github - appimagetoolurl = "https://github.com/probonopd/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) + # Download appimagetool from github + appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') os.chmod('appimagetool', 0o755) @@ -112,7 +112,7 @@ class Build(object): os.chdir('/tmp') def __del__(self): - del self.__builddir__ + #del self.__builddir__ if __name__ == '__main__': From e87d4085fbdba8a54870fea9bd90a94e80c94b5e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 00:53:43 +0100 Subject: [PATCH 025/176] Impostazione della classe per fare il build di tutte le versioni di base. --- build.py | 87 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 27 deletions(-) diff --git a/build.py b/build.py index 741c29f..9e1c140 100644 --- a/build.py +++ b/build.py @@ -2,7 +2,7 @@ import urllib.request from lxml import etree -import tempfile, os, sys, subprocess, shutil +import tempfile, os, sys, glob, subprocess, shutil class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] @@ -55,63 +55,96 @@ class Build(object): # * all languages, no help # * all languages + offline help - # Let's start with the quickest build: standard languages, no help. + # Preparation tasks + appnamedir = os.path.join(self.__builddir__, self.__appname__) + appimagedir = os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir') + os.makedirs(appimagedir, exist_ok = True) + # And then cd to the appname folder. + os.chdir(appnamedir) + # Download appimagetool from github + appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) + urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') + os.chmod('appimagetool', 0o755) + + # Run to build standard no help + self.__unpackbuild__('standard', False) + + # Run to build standard full help + self.__unpackbuild__('standard', True) + + # Build full no help + self.__unpackbuild__('full', False) + + # Full with help + self.__unpackbuild__('full', True) + + def __unpackbuild__(self, languageset = 'full', offlinehelp = False): # We start by filtering out tarballs from the list buildtarballs = [ self.__tarballs__[0] ] # Let's process standard languages and append results to the # buildtarball - for lang in Build.LANGSTD: - buildtarballs.extend([ x for x in self.__tarballs__ if ('langpack_' + lang) in x ]) - # If it was a build with offline help, another extend would have - # solved it: - #buildtarballs.extend([ x for x in self.__tarballs__ if ('helppack_'+ lang) in x ]) - - # Creating a subfolder - os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir'), exist_ok = True) - # And then cd to the appname folder. - os.chdir(os.path.join(self.__builddir__, self.__appname__)) - + if languageset == 'standard': + for lang in Build.LANGSTD: + buildtarballs.extend([ x for x in self.__tarballs__ if ('langpack_' + lang) in x ]) + if offlinehelp: + buildtarballs.extend([ x for x in self.__tarballs__ if ('helppack_' + lang) in x ]) + else: + # In any other cases, we build with all languages + if not offlinehelp: + buildtarballs.extend([ x for x in self.__tarballs__ if 'langpack_' in x ]) + else: + # We need also all help. Let's replace buildtarball with the + # whole bunch + buildtarballs = self.__tarballs__ + # Unpacking the tarballs for archive in buildtarballs: subprocess.run("tar xzf ../%s" % archive, shell=True) + os.chdir(appimagedir) # At this point, let's decompress the deb packages - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) - #subprocess.run("sed -i -e 's|Name=.*|Name=%s|g' startcenter.desktop" % self.__appname__, shell=True) - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) - os.makedirs(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin'), exist_ok = True) - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin')) + # Find the name of the binary called in the desktop file. + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') + + bindir=os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin') + os.makedirs(bindir, exist_ok = True) + os.chdir(bindir) subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) - os.chdir(os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir')) + os.chdir(appimagedir) # Download AppRun from github apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) urllib.request.urlretrieve(apprunurl, 'AppRun') os.chmod('AppRun', 0o755) - - os.chdir(os.path.join(self.__builddir__, self.__appname__)) - # Download appimagetool from github - appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) - urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') - os.chmod('appimagetool', 0o755) # Setting app version - appversion = self.__version__ + '.standard' + appversion = self.__version__ + '.' + languageset + if offlinehelp: + appversion += '.help' + # Building app subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) # Copying built image to final directory subprocess.run("find . -iname '*.AppImage' -exec cp {} %s \;" % outdir, shell = True) - os.chdir('/tmp') + + # Cleanup phase, before new run. + os.chdir(appnamedir) + for deb in glob.glob(appnamedir + '/*.deb'): + os.remove(deb) + shutil.rmtree(appimagedir) + os.makedirs(appimagedir) + def __del__(self): + """Destructor""" #del self.__builddir__ From 249527c78cc9d17c9259d43e2140e157233caecf Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:04:47 +0100 Subject: [PATCH 026/176] =?UTF-8?q?Terminata=20costruzione=20di=20tutte=20?= =?UTF-8?q?le=20build.=20Previsto=20checksum.=20AppImage=20sono=20ferme=20?= =?UTF-8?q?l=C3=AC.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.py | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/build.py b/build.py index 9e1c140..229c521 100644 --- a/build.py +++ b/build.py @@ -47,7 +47,7 @@ class Build(object): print("Got %s." % archive) - def build(self, outdir): + def build(self): """Building all the versions.""" # We have 4 builds to do: # * standard languages, no help @@ -56,11 +56,11 @@ class Build(object): # * all languages + offline help # Preparation tasks - appnamedir = os.path.join(self.__builddir__, self.__appname__) - appimagedir = os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir') - os.makedirs(appimagedir, exist_ok = True) + self.appnamedir = os.path.join(self.__builddir__, self.__appname__) + self.appimagedir = os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir') + os.makedirs(self.appimagedir, exist_ok = True) # And then cd to the appname folder. - os.chdir(appnamedir) + os.chdir(self.appnamedir) # Download appimagetool from github appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') @@ -102,7 +102,7 @@ class Build(object): for archive in buildtarballs: subprocess.run("tar xzf ../%s" % archive, shell=True) - os.chdir(appimagedir) + os.chdir(self.appimagedir) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file @@ -113,11 +113,11 @@ class Build(object): # Find the name of the binary called in the desktop file. binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') - bindir=os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir', 'usr', 'bin') + bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) os.chdir(bindir) subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) - os.chdir(appimagedir) + os.chdir(self.appimagedir) # Download AppRun from github apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) @@ -132,16 +132,28 @@ class Build(object): # Building app subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) - # Copying built image to final directory - subprocess.run("find . -iname '*.AppImage' -exec cp {} %s \;" % outdir, shell = True) + print("Built AppImage version {version}".format(version = appversion)) # Cleanup phase, before new run. - os.chdir(appnamedir) - for deb in glob.glob(appnamedir + '/*.deb'): + os.chdir(self.appnamedir) + for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - shutil.rmtree(appimagedir) - os.makedirs(appimagedir) + shutil.rmtree(self.appimagedir) + os.makedirs(self.appimagedir) + def checksums(self): + """Create checksums of the built versions.""" + os.chdir(self.appnamedir) + for appimage in glob.glob('*.AppImage'): + # See if a checksum already exist + if not os.path.exists(appimage + '.md5'): + subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) + + + def move(self, outdir): + """Moves built versions to definitive storage.""" + os.chdir(self.appnamedir) + subprocess.run("find . -iname '*.AppImage*' -exec cp {} %s \;" % outdir, shell=True) def __del__(self): """Destructor""" @@ -161,5 +173,6 @@ if __name__ == '__main__': b = Build(sys.argv[1], sys.argv[2], sys.argv[3]) b.download() - b.build(sys.argv[4]) + b.build() + b.checksums() del b From a059e27a205d0a5da5e490e5758ea1e7c919a7b6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:09:53 +0100 Subject: [PATCH 027/176] Corretto flusso di rebuild. --- build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 229c521..83b48b8 100644 --- a/build.py +++ b/build.py @@ -117,9 +117,9 @@ class Build(object): os.makedirs(bindir, exist_ok = True) os.chdir(bindir) subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) - os.chdir(self.appimagedir) # Download AppRun from github + os.chdir(self.appimagedir) apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) urllib.request.urlretrieve(apprunurl, 'AppRun') os.chmod('AppRun', 0o755) @@ -130,6 +130,7 @@ class Build(object): appversion += '.help' # Building app + os.chdir(self.appnamedir) subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) print("Built AppImage version {version}".format(version = appversion)) From 39ad25ab2f9f2a154d825f6b5cd1a6ef63d13689 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:34:37 +0100 Subject: [PATCH 028/176] Created a download folder to speed up rebuilds. --- build.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/build.py b/build.py index 83b48b8..5ba4f23 100644 --- a/build.py +++ b/build.py @@ -7,11 +7,12 @@ import tempfile, os, sys, glob, subprocess, shutil class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] - def __init__(self, query, arch, url): + def __init__(self, query, arch, url, downloaddir = '/var/tmp/downloads'): """Build all versions that can be found in the indicated repo.""" self.__query__ = query self.__arch__ = arch self.__url__ = url + self.__downloaddir__ = downloaddir # Creating a tempfile self.__builddir__ = tempfile.mkdtemp() @@ -36,8 +37,14 @@ class Build(object): self.__appname__ = main_arr[0] self.__version__ = main_arr[1] - os.chdir(self.__builddir__) + os.makedirs(self.__downloaddir__, exist_ok = True) + os.chdir(self.__downloaddir__) for archive in self.__tarballs__: + # If the archive is already there, do not do anything. + if os.path.exists(os.path.join(self.__downloaddir__, archive)): + print("Archive %s is already there! Sweet") + continue + # Download the archive try: urllib.request.urlretrieve(self.__url__ + archive, archive) @@ -100,13 +107,14 @@ class Build(object): # Unpacking the tarballs for archive in buildtarballs: - subprocess.run("tar xzf ../%s" % archive, shell=True) + subprocess.run("tar xzf {folder}/{archive}".format(folder = self.__downloaddir__, archive = archive), shell=True) os.chdir(self.appimagedir) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) + subprocess.run("sed -i -e ", shell=True) subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) @@ -139,7 +147,7 @@ class Build(object): os.chdir(self.appnamedir) for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - shutil.rmtree(self.appimagedir) + subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) os.makedirs(self.appimagedir) def checksums(self): From f6488c2e4f7f7eb491d443e737a07f118b21705a Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:42:39 +0100 Subject: [PATCH 029/176] Correzione nome dell'applicazione, correzione della versione. --- build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 5ba4f23..249b6bb 100644 --- a/build.py +++ b/build.py @@ -114,7 +114,9 @@ class Build(object): subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) - subprocess.run("sed -i -e ", shell=True) + appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:" % appname, shell=True) + subprocess.run("sed -i -e 's:^Version=.*$:Version=%s:" % self.__version__, shell=True) subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) From 14e592d1f2a646f2f85d35fd3afd4f26b9799ad7 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:45:24 +0100 Subject: [PATCH 030/176] Correzione nome dell'applicazione (2). --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 249b6bb..a42084a 100644 --- a/build.py +++ b/build.py @@ -115,8 +115,8 @@ class Build(object): # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:" % appname, shell=True) - subprocess.run("sed -i -e 's:^Version=.*$:Version=%s:" % self.__version__, shell=True) + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True) + subprocess.run("sed -i -e 's:^Version=.*$:Version=%s:' startcenter.desktop" % self.__version__, shell=True) subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) From 052b8c45eaa18a3b28c41b7982d12d2961662f23 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:47:40 +0100 Subject: [PATCH 031/176] Non correggo la versione per non spaccare la build. --- build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build.py b/build.py index a42084a..8f0b24e 100644 --- a/build.py +++ b/build.py @@ -116,7 +116,6 @@ class Build(object): subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True) - subprocess.run("sed -i -e 's:^Version=.*$:Version=%s:' startcenter.desktop" % self.__version__, shell=True) subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) From c935c1a3d05624dcb1a01372320a529ea64fa871 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 01:57:20 +0100 Subject: [PATCH 032/176] Attivata rimozione cartella di build, attivato spostamento delle App in Storage. --- build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 8f0b24e..f9757a4 100644 --- a/build.py +++ b/build.py @@ -167,7 +167,7 @@ class Build(object): def __del__(self): """Destructor""" - #del self.__builddir__ + del self.__builddir__ if __name__ == '__main__': @@ -185,4 +185,5 @@ if __name__ == '__main__': b.download() b.build() b.checksums() + b.move(sys.argv[4]) del b From aa0d223670cffb6af5956ed45fb697239bd509cc Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 02:43:27 +0100 Subject: [PATCH 033/176] Problemi con build dell'immagine. Rimuovo temporaneamente tutte le build e la rimozione della cartella. --- build.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/build.py b/build.py index f9757a4..6ff1393 100644 --- a/build.py +++ b/build.py @@ -77,13 +77,13 @@ class Build(object): self.__unpackbuild__('standard', False) # Run to build standard full help - self.__unpackbuild__('standard', True) + #self.__unpackbuild__('standard', True) # Build full no help - self.__unpackbuild__('full', False) + #self.__unpackbuild__('full', False) # Full with help - self.__unpackbuild__('full', True) + #self.__unpackbuild__('full', True) def __unpackbuild__(self, languageset = 'full', offlinehelp = False): # We start by filtering out tarballs from the list @@ -109,29 +109,28 @@ class Build(object): for archive in buildtarballs: subprocess.run("tar xzf {folder}/{archive}".format(folder = self.__downloaddir__, archive = archive), shell=True) - os.chdir(self.appimagedir) + os.chdir(self.appnamedir) # At this point, let's decompress the deb packages - subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True) + subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) # Changing desktop file - subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True) + subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True) + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True, cwd=self.appimagedir) - subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True) + subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) # Find the name of the binary called in the desktop file. - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True).decode('utf-8').strip('\n') + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - os.chdir(bindir) - subprocess.run("find ../../opt -name soffice -path '*programm*' -exec ln -s {} %s \;" % binaryname, shell=True) + subprocess.run("find ../../opt -iname soffice -path '*programm*' -exec ln -sf {} %s \;" % binaryname, shell=True, cwd=bindir) # Download AppRun from github - os.chdir(self.appimagedir) apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) - urllib.request.urlretrieve(apprunurl, 'AppRun') - os.chmod('AppRun', 0o755) + dest = os.path.join(self.appimagedir, 'AppRun') + urllib.request.urlretrieve(apprunurl, dest) + os.chmod(dest, 0o755) # Setting app version appversion = self.__version__ + '.' + languageset @@ -139,13 +138,11 @@ class Build(object): appversion += '.help' # Building app - os.chdir(self.appnamedir) subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) print("Built AppImage version {version}".format(version = appversion)) # Cleanup phase, before new run. - os.chdir(self.appnamedir) for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) @@ -167,7 +164,7 @@ class Build(object): def __del__(self): """Destructor""" - del self.__builddir__ + #del self.__builddir__ if __name__ == '__main__': From 84452b4abae9aa72ef6a7755b2a2777444ac89b8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 03:01:32 +0100 Subject: [PATCH 034/176] Ancora tentativi di correzione della build. --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 6ff1393..7d93f73 100644 --- a/build.py +++ b/build.py @@ -42,7 +42,7 @@ class Build(object): for archive in self.__tarballs__: # If the archive is already there, do not do anything. if os.path.exists(os.path.join(self.__downloaddir__, archive)): - print("Archive %s is already there! Sweet") + print("Archive %s is already there! Sweet" % archive) continue # Download the archive @@ -124,7 +124,7 @@ class Build(object): bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run("find ../../opt -iname soffice -path '*programm*' -exec ln -sf {} %s \;" % binaryname, shell=True, cwd=bindir) + subprocess.run("find ../../opt -iname soffice -path '*programm*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) # Download AppRun from github apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) From 0fcae9c1640acd375efc2f1111d55fc232bc819e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 03:14:56 +0100 Subject: [PATCH 035/176] Trovato e risolto problema di build. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 7d93f73..9e2bf2a 100644 --- a/build.py +++ b/build.py @@ -124,7 +124,7 @@ class Build(object): bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run("find ../../opt -iname soffice -path '*programm*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) + subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) # Download AppRun from github apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) From 1508540847e8b5dfecc931a4ad08bedca9c2e438 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 03:21:22 +0100 Subject: [PATCH 036/176] Rimesso tutto in carreggiata. Build dovrebbero tutte prodursi al prossimo giro. --- build.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index 9e2bf2a..b0b836f 100644 --- a/build.py +++ b/build.py @@ -77,13 +77,13 @@ class Build(object): self.__unpackbuild__('standard', False) # Run to build standard full help - #self.__unpackbuild__('standard', True) + self.__unpackbuild__('standard', True) # Build full no help - #self.__unpackbuild__('full', False) + self.__unpackbuild__('full', False) # Full with help - #self.__unpackbuild__('full', True) + self.__unpackbuild__('full', True) def __unpackbuild__(self, languageset = 'full', offlinehelp = False): # We start by filtering out tarballs from the list @@ -112,6 +112,7 @@ class Build(object): os.chdir(self.appnamedir) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + os.makedirs(self.appimagedir, exist_ok = True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' @@ -146,7 +147,6 @@ class Build(object): for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) - os.makedirs(self.appimagedir) def checksums(self): """Create checksums of the built versions.""" @@ -164,7 +164,7 @@ class Build(object): def __del__(self): """Destructor""" - #del self.__builddir__ + del self.__builddir__ if __name__ == '__main__': From 27e004a84b125d1336f6b8ffcb5db3d76e15761f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 03:49:31 +0100 Subject: [PATCH 037/176] Piccolo errore di ordine di comandi. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index b0b836f..7ae68eb 100644 --- a/build.py +++ b/build.py @@ -110,9 +110,9 @@ class Build(object): subprocess.run("tar xzf {folder}/{archive}".format(folder = self.__downloaddir__, archive = archive), shell=True) os.chdir(self.appnamedir) + os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) - os.makedirs(self.appimagedir, exist_ok = True) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' From c153362a0602d4b842c07dcf000f4ac0795aaf9c Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 04:25:16 +0100 Subject: [PATCH 038/176] Implementata pulizia builddir. --- build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 7ae68eb..1dda163 100644 --- a/build.py +++ b/build.py @@ -164,7 +164,8 @@ class Build(object): def __del__(self): """Destructor""" - del self.__builddir__ + # Cleaning up build directory + shutil.rmtree(self.__builddir__) if __name__ == '__main__': From c79cc2abcf9d4bb7aa56ad7a88060f92158b4320 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 14:33:07 +0100 Subject: [PATCH 039/176] Preparato layout cartelle per libreria. --- .../loappimage-helpers/__init__.py | 0 .../loappimage-helpers/versions.py | 0 scripts/loaih-getversion | 88 +++++++++++++++++++ 3 files changed, 88 insertions(+) rename build.py => lib/loappimage-helpers/__init__.py (100%) rename getversions.py => lib/loappimage-helpers/versions.py (100%) create mode 100644 scripts/loaih-getversion diff --git a/build.py b/lib/loappimage-helpers/__init__.py similarity index 100% rename from build.py rename to lib/loappimage-helpers/__init__.py diff --git a/getversions.py b/lib/loappimage-helpers/versions.py similarity index 100% rename from getversions.py rename to lib/loappimage-helpers/versions.py diff --git a/scripts/loaih-getversion b/scripts/loaih-getversion new file mode 100644 index 0000000..b6acdd8 --- /dev/null +++ b/scripts/loaih-getversion @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import urllib.request +from lxml import etree +from packaging.version import parse as parse_version +import re, sys, json + +ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" +RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" +DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/current/" + +def getlatestrel(basever): + """Search in downloadarchive for the latest version matching baseversion.""" + versionlist = etree.HTML(urllib.request.urlopen(ARCHIVE).read()).xpath('//td/a') + # Getting a more polished matching list + cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) + + # Sorting, then returning the last version + return sorted(cleanlist)[-1] + +def getbranchrel(branch): + """Based on branch names, get the release number.""" + basedirurl = {} + version = '' + if branch == 'daily': + basedirurl = { 'x86_64': DAILY, 'x86': '-' } + version = etree.HTML(urllib.request.urlopen(DAILY).read()).xpath('//td/a')[1].text.split('_')[1] + + return { 'version': version, 'basedirurl': basedirurl } + + versions = etree.HTML(urllib.request.urlopen(RELEASE).read()).xpath('//td/a') + index = 1 + if branch == 'still': + index = 2 + elif branch == 'fresh': + index = 3 + version = getlatestrel(versions[index].text.strip('/')) + + return { 'version': version, 'basedirurl': getbaseurl(version) } + +def getbaseurl(version): + """Returns the links based on the numeric version.""" + basediriurl = {} + url = ARCHIVE + '/' + version + '/deb/' + # x86 binaries are not anymore offered after 6.3.0. + if parse_version(version) < parse_version('6.3.0'): + basedirurl['x86'] = url + 'x86/' + else: + basedirurl['x86'] = '-' + + basedirurl['x86_64'] = url + 'x86_64/' + + return basedirurl + +if __name__ == '__main__': + # Preparing variables for outputting + version = '' + basedirurl = {} + basever = 'fresh' + + # At the end of the checks, we need a version string and a basedirurl, which + # should be a dictionaly for x86, x86_64 with the base of the directory where + # to download the files. + if len(sys.argv) > 1: + # A version has been specified. + basever = sys.argv[1] + + # Once overridden with Argv, parse the inputs + if '.' in basever: + # Numbered version. Let's check it is a 4 dotted release + if len(basever.split('.')) == 4: + version = basever + else: + version = getlatestrel(basever) + + basedirurl = getbaseurl(version) + else: + # String versions. + a = getbranchrel(basever) + version = a['version'] + basedirurl = a['basedirurl'] + + output = """RUNDECK:DATA: query = %s +RUNDECK:DATA: version = %s +RUNDECK:DATA: x86 = %s +RUNDECK:DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) +print(output) From 6aa0b34216c4c10ee7345dae88c5b445c6198a57 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 20 Mar 2022 23:57:45 +0100 Subject: [PATCH 040/176] Prima versione riscritta. --- lib/loappimage-helpers/__init__.py | 187 ----------------------- lib/loappimage-helpers/versions.py | 88 ----------- loaih/__init__.py | 230 +++++++++++++++++++++++++++++ loaih/versions.py | 76 ++++++++++ scripts/loaih-build | 42 ++++++ scripts/loaih-getversion | 102 +++---------- setup.py | 18 +++ 7 files changed, 390 insertions(+), 353 deletions(-) delete mode 100644 lib/loappimage-helpers/__init__.py delete mode 100644 lib/loappimage-helpers/versions.py create mode 100644 loaih/__init__.py create mode 100644 loaih/versions.py create mode 100644 scripts/loaih-build create mode 100644 setup.py diff --git a/lib/loappimage-helpers/__init__.py b/lib/loappimage-helpers/__init__.py deleted file mode 100644 index 1dda163..0000000 --- a/lib/loappimage-helpers/__init__.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 - -import urllib.request -from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil - -class Build(object): - LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] - - def __init__(self, query, arch, url, downloaddir = '/var/tmp/downloads'): - """Build all versions that can be found in the indicated repo.""" - self.__query__ = query - self.__arch__ = arch - self.__url__ = url - self.__downloaddir__ = downloaddir - - # Creating a tempfile - self.__builddir__ = tempfile.mkdtemp() - self.__tarballs__ = [] - self.__appname__ = '' - self.__version__ = '' - - if self.__url__ == '-': - print("Cannot find this version for arch {arch}.".format(arch = self.__arch__)) - return False - - - def download(self): - """Downloads the contents of the URL as it was a folder.""" - - # Let's start with defining which files are to be downloaded. - # Let's explore the remote folder. - contents = etree.HTML(urllib.request.urlopen(self.__url__).read()).xpath("//td/a") - self.__tarballs__ = [ x.text for x in contents if x.text.endswith('tar.gz') ] - maintarball = self.__tarballs__[0] - main_arr = maintarball.split('_') - self.__appname__ = main_arr[0] - self.__version__ = main_arr[1] - - os.makedirs(self.__downloaddir__, exist_ok = True) - os.chdir(self.__downloaddir__) - for archive in self.__tarballs__: - # If the archive is already there, do not do anything. - if os.path.exists(os.path.join(self.__downloaddir__, archive)): - print("Archive %s is already there! Sweet" % archive) - continue - - # Download the archive - try: - urllib.request.urlretrieve(self.__url__ + archive, archive) - except: - print("Failed to download {archive}.".format(archive = archive)) - - print("Got %s." % archive) - - - def build(self): - """Building all the versions.""" - # We have 4 builds to do: - # * standard languages, no help - # * standard languages + offline help - # * all languages, no help - # * all languages + offline help - - # Preparation tasks - self.appnamedir = os.path.join(self.__builddir__, self.__appname__) - self.appimagedir = os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir') - os.makedirs(self.appimagedir, exist_ok = True) - # And then cd to the appname folder. - os.chdir(self.appnamedir) - # Download appimagetool from github - appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) - urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') - os.chmod('appimagetool', 0o755) - - # Run to build standard no help - self.__unpackbuild__('standard', False) - - # Run to build standard full help - self.__unpackbuild__('standard', True) - - # Build full no help - self.__unpackbuild__('full', False) - - # Full with help - self.__unpackbuild__('full', True) - - def __unpackbuild__(self, languageset = 'full', offlinehelp = False): - # We start by filtering out tarballs from the list - buildtarballs = [ self.__tarballs__[0] ] - - # Let's process standard languages and append results to the - # buildtarball - if languageset == 'standard': - for lang in Build.LANGSTD: - buildtarballs.extend([ x for x in self.__tarballs__ if ('langpack_' + lang) in x ]) - if offlinehelp: - buildtarballs.extend([ x for x in self.__tarballs__ if ('helppack_' + lang) in x ]) - else: - # In any other cases, we build with all languages - if not offlinehelp: - buildtarballs.extend([ x for x in self.__tarballs__ if 'langpack_' in x ]) - else: - # We need also all help. Let's replace buildtarball with the - # whole bunch - buildtarballs = self.__tarballs__ - - # Unpacking the tarballs - for archive in buildtarballs: - subprocess.run("tar xzf {folder}/{archive}".format(folder = self.__downloaddir__, archive = archive), shell=True) - - os.chdir(self.appnamedir) - os.makedirs(self.appimagedir, exist_ok = True) - # At this point, let's decompress the deb packages - subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) - # Changing desktop file - subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) - appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True, cwd=self.appimagedir) - - subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) - - # Find the name of the binary called in the desktop file. - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') - - bindir=os.path.join(self.appimagedir, 'usr', 'bin') - os.makedirs(bindir, exist_ok = True) - subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) - - # Download AppRun from github - apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) - dest = os.path.join(self.appimagedir, 'AppRun') - urllib.request.urlretrieve(apprunurl, dest) - os.chmod(dest, 0o755) - - # Setting app version - appversion = self.__version__ + '.' + languageset - if offlinehelp: - appversion += '.help' - - # Building app - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) - - print("Built AppImage version {version}".format(version = appversion)) - - # Cleanup phase, before new run. - for deb in glob.glob(self.appnamedir + '/*.deb'): - os.remove(deb) - subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) - - def checksums(self): - """Create checksums of the built versions.""" - os.chdir(self.appnamedir) - for appimage in glob.glob('*.AppImage'): - # See if a checksum already exist - if not os.path.exists(appimage + '.md5'): - subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) - - - def move(self, outdir): - """Moves built versions to definitive storage.""" - os.chdir(self.appnamedir) - subprocess.run("find . -iname '*.AppImage*' -exec cp {} %s \;" % outdir, shell=True) - - def __del__(self): - """Destructor""" - # Cleaning up build directory - shutil.rmtree(self.__builddir__) - - -if __name__ == '__main__': - # Run if it is run as a program. - # 1 -> query - # 2 -> arch - # 3 -> url - # 4 -> outdir - - if not len(sys.argv) == 5: - print("Please launch with this parameters: build.py query arch url outputdir") - sys.exit(255) - - b = Build(sys.argv[1], sys.argv[2], sys.argv[3]) - b.download() - b.build() - b.checksums() - b.move(sys.argv[4]) - del b diff --git a/lib/loappimage-helpers/versions.py b/lib/loappimage-helpers/versions.py deleted file mode 100644 index b6acdd8..0000000 --- a/lib/loappimage-helpers/versions.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import urllib.request -from lxml import etree -from packaging.version import parse as parse_version -import re, sys, json - -ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" -RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" -DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/current/" - -def getlatestrel(basever): - """Search in downloadarchive for the latest version matching baseversion.""" - versionlist = etree.HTML(urllib.request.urlopen(ARCHIVE).read()).xpath('//td/a') - # Getting a more polished matching list - cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) - - # Sorting, then returning the last version - return sorted(cleanlist)[-1] - -def getbranchrel(branch): - """Based on branch names, get the release number.""" - basedirurl = {} - version = '' - if branch == 'daily': - basedirurl = { 'x86_64': DAILY, 'x86': '-' } - version = etree.HTML(urllib.request.urlopen(DAILY).read()).xpath('//td/a')[1].text.split('_')[1] - - return { 'version': version, 'basedirurl': basedirurl } - - versions = etree.HTML(urllib.request.urlopen(RELEASE).read()).xpath('//td/a') - index = 1 - if branch == 'still': - index = 2 - elif branch == 'fresh': - index = 3 - version = getlatestrel(versions[index].text.strip('/')) - - return { 'version': version, 'basedirurl': getbaseurl(version) } - -def getbaseurl(version): - """Returns the links based on the numeric version.""" - basediriurl = {} - url = ARCHIVE + '/' + version + '/deb/' - # x86 binaries are not anymore offered after 6.3.0. - if parse_version(version) < parse_version('6.3.0'): - basedirurl['x86'] = url + 'x86/' - else: - basedirurl['x86'] = '-' - - basedirurl['x86_64'] = url + 'x86_64/' - - return basedirurl - -if __name__ == '__main__': - # Preparing variables for outputting - version = '' - basedirurl = {} - basever = 'fresh' - - # At the end of the checks, we need a version string and a basedirurl, which - # should be a dictionaly for x86, x86_64 with the base of the directory where - # to download the files. - if len(sys.argv) > 1: - # A version has been specified. - basever = sys.argv[1] - - # Once overridden with Argv, parse the inputs - if '.' in basever: - # Numbered version. Let's check it is a 4 dotted release - if len(basever.split('.')) == 4: - version = basever - else: - version = getlatestrel(basever) - - basedirurl = getbaseurl(version) - else: - # String versions. - a = getbranchrel(basever) - version = a['version'] - basedirurl = a['basedirurl'] - - output = """RUNDECK:DATA: query = %s -RUNDECK:DATA: version = %s -RUNDECK:DATA: x86 = %s -RUNDECK:DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) -print(output) diff --git a/loaih/__init__.py b/loaih/__init__.py new file mode 100644 index 0000000..76afa7b --- /dev/null +++ b/loaih/__init__.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 + +import urllib.request +import loaih.versions as versions +from lxml import etree +import tempfile, os, sys, glob, subprocess, shutil + +class Build(object): + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] + LANGBASIC = [ 'en-GB' ] + + def __init__(self, query, arch): + """Build all versions that can be found in the indicated repo.""" + self.query = query + self.arch = arch + self.url = '' + self.language = 'basic' + self.offline_help = False + self.portable = False + self.updatable = True + self.storage_path = '/srv/http/appimage.sys42.eu' + self.download_path = '/var/tmp/downloads' + + # Getting versions and so on + v = versions.BuildVersion(self.query) + + # Building expected AppImageName + languagepart = "." + if ',' in self.language: + languagepart += self.language.replace(',', '-') + + self.appimagefilename['x86'] = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + '-' + v.version + languagepart + '.help' if self.offline_help else '' + '-x86.AppImage' + self.appimagefilename['x86_64'] = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + '-' + v.version + languagepart + '.help' if self.offline_help else '' + '-x86_64.AppImage' + + # Creating a tempfile + self.builddir = tempfile.mkdtemp() + self.tarballs = [] + self.appname = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + self.version = v.version + self.url = v.basedirurl + self.built = False + + def check(self, storage_path): + """Checking if the requested AppImage has been already built.""" + self.storage_path = storage_path + + if self.portable: + self.storage_path += "/portable" + + for arch in self.arch: + res = subprocess.check_output("find {path} -name '{appimage}'".format( + path = self.storage_path, + appimage = self.appimagefilename[arch] + '.zsync' if self.updatable else '' + ), shell=True).decode('utf-8').strip('\n') + if len(res) > 1: + self.built = True + + if self.built: + print("The requested AppImage already exists on storage. I'll skip downloading, building and moving the results.") + + def download(self, download_path): + """Downloads the contents of the URL as it was a folder.""" + if self.built: + return + + # Let's start with defining which files are to be downloaded. + # Let's explore the remote folder. + self.download_path = download_path + + for a in self.arch: + contents = etree.HTML(urllib.request.urlopen(self.url[a]).read()).xpath("//td/a") + self.tarballs = [ x.text for x in contents if x.text.endswith('tar.gz') ] + maintarball = self.tarballs[0] + mainarr = maintarball.split('') + self.appname = mainarr[0] + self.version = mainarr[1] + + os.makedirs(self.downloaddir, existok = True) + os.chdir(self.downloaddir) + for archive in self.tarballs: + # If the archive is already there, do not do anything. + if os.path.exists(os.path.join(self.downloaddir, archive)): + print("Archive %s is already there! Sweet" % archive) + continue + + # Download the archive + try: + urllib.request.urlretrieve(self.url + archive, archive) + except: + print("Failed to download {archive}.".format(archive = archive)) + + print("Got %s." % archive) + + def build(self): + """Building all the versions.""" + if self.built: + return + + # We have 4 builds to do: + # * standard languages, no help + # * standard languages + offline help + # * all languages, no help + # * all languages + offline help + + if self.portable and not 'portable' in self.storage_path: + self.storage_path += "/portable" + + # Incompatibilities - if portable and updatable are asked together, + # only portable will be built. + if self.portable and self.updatable: + print("Upgradable and portable options were required together. Building only portable.") + self.updatable = False + + for arch in self.arch: + # Preparation tasks + self.appnamedir = os.path.join(self.builddir, self.appname) + self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') + os.makedirs(self.appimagedir, exist_ok = True) + # And then cd to the appname folder. + os.chdir(self.appnamedir) + # Download appimagetool from github + appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) + urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') + os.chmod('appimagetool', 0o755) + + # Build the requested version. + self.__unpackbuild__() + + def __unpackbuild__(self, languageset = 'full', offlinehelp = False): + # We start by filtering out tarballs from the list + buildtarballs = [ self.__tarballs__[0] ] + + # Let's process standard languages and append results to the + # buildtarball + if self.language == 'basic': + buildtarballs.extend([ x for x in self.tarballs if 'langpack_en-GB' in x]) + if self.offline_help: + buildtarballs.extend([ x for x in self.tarballs if 'helppack_en-GB' in x ]) + if self.language == 'standard': + for lang in Build.LANGSTD: + buildtarballs.extend([ x for x in self.tarballs if ('langpack' + lang) in x ]) + if self.offlinehelp: + buildtarballs.extend([ x for x in self.tarballs if ('helppack' + lang) in x ]) + else: + # In any other cases, we build with all languages + if not self.offlinehelp: + buildtarballs.extend([ x for x in self.tarballs if 'langpack' in x ]) + else: + # We need also all help. Let's replace buildtarball with the + # whole bunch + buildtarballs = self.tarballs + + # Unpacking the tarballs + for archive in buildtarballs: + subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) + + os.chdir(self.appnamedir) + os.makedirs(self.appimagedir, exist_ok = True) + # At this point, let's decompress the deb packages + subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + # Changing desktop file + subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) + appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True, cwd=self.appimagedir) + + subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) + + # Find the name of the binary called in the desktop file. + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') + + bindir=os.path.join(self.appimagedir, 'usr', 'bin') + os.makedirs(bindir, exist_ok = True) + subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) + + # Download AppRun from github + apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) + dest = os.path.join(self.appimagedir, 'AppRun') + urllib.request.urlretrieve(apprunurl, dest) + os.chmod(dest, 0o755) + + # Setting app version + appversion = self.version + '.' + self.language if not ',' in self.language else self.language.replace(',', '-') + if self.offline_help: + appversion += '.help' + + # Building app + if self.updatable: + zsync = self.appimagefilename[arch] + '.zsync' + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = zsync, appname = self.__appname__), shell=True) + + else: + subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.appname), shell=True) + + print("Built AppImage version {version}".format(version = appversion)) + + # Cleanup phase, before new run. + for deb in glob.glob(self.appnamedir + '/*.deb'): + os.remove(deb) + subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) + + def checksums(self): + """Create checksums of the built versions.""" + if self.built: + return + + os.chdir(self.appnamedir) + for appimage in glob.glob('*.AppImage*'): + # See if a checksum already exist + if not os.path.exists(appimage + '.md5'): + subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) + + + def move(self, outdir): + """Moves built versions to definitive storage.""" + if self.built: + return + if not outdir in self.storage_path: + # Let's create an updated storage_path + if self.portable: + self.storage_path = outdir + '/portable' + else: + self.storage_path = outdir + + os.chdir(self.appnamedir) + subprocess.run("find . -iname '*.AppImage*' -exec cp {} %s \;" % self.storage_path, shell=True) + + def __del__(self): + """Destructor""" + # Cleaning up build directory + shutil.rmtree(self.builddir) diff --git a/loaih/versions.py b/loaih/versions.py new file mode 100644 index 0000000..7def95d --- /dev/null +++ b/loaih/versions.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import urllib.request +from lxml import etree +from packaging.version import parse as parse_version + +class BuildVersion(object): + ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" + RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" + DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/current/" + + def __init__(self, query): + self.query = query + self.version = '' + self.basedirurl = {} + + # Parsing the query input. + if '.' in self.query: + # Numbered self.version. Let's check it is a 4 dotted release + if len(self.query.split('.')) == 4: + self.version = self.query + else: + # If not 4 dotted, let's search for the 4 dotted version + self.version = self.__getlatestrel(self.query) + + self.basedirurl = self.__getbaseurl(self.version) + else: + # String self.versions. + a = self.__getbranchrel(self.query) + self.version = a['version'] + self.basedirurl = a['basedirurl'] + + def __getlatestrel(self, basever): + """Search in downloadarchive for the latest version matching baseversion.""" + versionlist = etree.HTML(urllib.request.urlopen(BuildVersion.ARCHIVE).read()).xpath('//td/a') + # Getting a more polished matching list + cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) + + # Sorting, then returning the last version + return sorted(cleanlist)[-1] + + def __getbranchrel(self, branch): + """Based on branch names, get the release number.""" + basedirurl = {} + version = '' + if branch == 'daily': + basedirurl = { 'x86_64': BuildVersion.DAILY, 'x86': '-' } + version = etree.HTML(urllib.request.urlopen(BuildVersion.DAILY).read()).xpath('//td/a')[1].text.split('_')[1] + + return { 'version': version, 'basedirurl': basedirurl } + + versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') + index = 1 + if branch == 'still': + index = 2 + elif branch == 'fresh': + index = 3 + version = self.__getlatestrel(versions[index].text.strip('/')) + + return { 'version': version, 'basedirurl': self.__getbaseurl(version) } + + def __getbaseurl(self, version): + """Returns the links based on the numeric version.""" + basedirurl = {} + url = BuildVersion.ARCHIVE + '/' + version + '/deb/' + + # x86 binaries are not anymore offered after 6.3.0. + if parse_version(version) < parse_version('6.3.0'): + basedirurl['x86'] = url + 'x86/' + else: + basedirurl['x86'] = '-' + + basedirurl['x86_64'] = url + 'x86_64/' + + return basedirurl diff --git a/scripts/loaih-build b/scripts/loaih-build new file mode 100644 index 0000000..8392b37 --- /dev/null +++ b/scripts/loaih-build @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import click +import loaih + +@click.command() +@click.option('-a', '--arch', 'arch', type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), default='all', help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") +@click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") +@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") +@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") +@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") +@click.option('-d', '--download-path', 'download', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") +@click.option('-s', '--storage-path', 'storage', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") +@click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") +@click.argument('query') +def build(arch, language, offline, portable, updatable, download, storage, check, query): + # Parsing options + if arch.lower() == 'all': + # We need to build it twice. + arches = [ 'x86', 'x86_64' ] + else: + arches = [ arch.lower() ] + + for arch in arches: + obj = loaih.build(query, arch) + + obj.language = language + obj.offline_help = offlinehelp + obj.portable = portable + obj.updatable = updatable + + if check: + obj.check(storage) + obj.download(download) + obj.build() + obj.checksums() + obj.move(storage) + del obj + +if __name__ == '__main__': + build() diff --git a/scripts/loaih-getversion b/scripts/loaih-getversion index b6acdd8..59d03ae 100644 --- a/scripts/loaih-getversion +++ b/scripts/loaih-getversion @@ -1,88 +1,34 @@ #!/usr/bin/env python # encoding: utf-8 -import urllib.request -from lxml import etree -from packaging.version import parse as parse_version +import click +from loaih.versions import BuildVersion import re, sys, json -ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" -RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" -DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/current/" +@click.command() +@click.option('-o', '--output', default = 'rundeck', type=click.Choice(['rundeck', 'json', 'text' ], case_sensitive=False), help="Output format, defaulting to Rundeck Key/Value data format. Options: rundeck,json,text") +@click.argument('query') +def getversion(query, output): + b = BuildVersion(query) -def getlatestrel(basever): - """Search in downloadarchive for the latest version matching baseversion.""" - versionlist = etree.HTML(urllib.request.urlopen(ARCHIVE).read()).xpath('//td/a') - # Getting a more polished matching list - cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) - - # Sorting, then returning the last version - return sorted(cleanlist)[-1] - -def getbranchrel(branch): - """Based on branch names, get the release number.""" - basedirurl = {} - version = '' - if branch == 'daily': - basedirurl = { 'x86_64': DAILY, 'x86': '-' } - version = etree.HTML(urllib.request.urlopen(DAILY).read()).xpath('//td/a')[1].text.split('_')[1] - - return { 'version': version, 'basedirurl': basedirurl } - - versions = etree.HTML(urllib.request.urlopen(RELEASE).read()).xpath('//td/a') - index = 1 - if branch == 'still': - index = 2 - elif branch == 'fresh': - index = 3 - version = getlatestrel(versions[index].text.strip('/')) - - return { 'version': version, 'basedirurl': getbaseurl(version) } - -def getbaseurl(version): - """Returns the links based on the numeric version.""" - basediriurl = {} - url = ARCHIVE + '/' + version + '/deb/' - # x86 binaries are not anymore offered after 6.3.0. - if parse_version(version) < parse_version('6.3.0'): - basedirurl['x86'] = url + 'x86/' + if output.lower() == 'rundeck': + print("""RUNDECK:DATA: query = {query} +RUNDECK:DATA: version = {version} +RUNDECK:DATA: x86 = {x86_url} +RUNDECK:DATA: x86_64 = {x86_64_url}""".format(query = query, version = b.version, x86_url = b.basedirurl['x86'], x86_64_url = b.basedirurl['x86_64'])) + elif output.lower() == 'json': + output = { + 'query': query, + 'version': b.version, + 'basedirurl': b.basedirurl + } + print(json.dumps(output)) else: - basedirurl['x86'] = '-' - - basedirurl['x86_64'] = url + 'x86_64/' + print("""query: {query} +version: {version} +x86: {x86_url} +x86_64: {x86_64_url}""".format(query = query, version = b.version, x86_url = b.basedirurl['x86'], x86_64_url = b.basedirurl['x86_64'])) - return basedirurl if __name__ == '__main__': - # Preparing variables for outputting - version = '' - basedirurl = {} - basever = 'fresh' - - # At the end of the checks, we need a version string and a basedirurl, which - # should be a dictionaly for x86, x86_64 with the base of the directory where - # to download the files. - if len(sys.argv) > 1: - # A version has been specified. - basever = sys.argv[1] - - # Once overridden with Argv, parse the inputs - if '.' in basever: - # Numbered version. Let's check it is a 4 dotted release - if len(basever.split('.')) == 4: - version = basever - else: - version = getlatestrel(basever) - - basedirurl = getbaseurl(version) - else: - # String versions. - a = getbranchrel(basever) - version = a['version'] - basedirurl = a['basedirurl'] - - output = """RUNDECK:DATA: query = %s -RUNDECK:DATA: version = %s -RUNDECK:DATA: x86 = %s -RUNDECK:DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) -print(output) + getversion() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0f9a07d --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# encoding: utf-8 +# vim:sts=4:sw=4 + +from setuptools import setup,find_packages + +setup( + name="loaih", + version="1.0.0", + description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", + author="Emiliano Vavassori", + author_email="syntaxerrormmm@libreoffice.org", + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + scripts=[ 'scripts/loaih-getversion', 'scripts/loaih-build' ], + install_requires=[ 'click', ], + license='MIT', + url='https://git.libreitalia.org/LibreItalia/loappimage-helpers/', +) From 564a933f65335574b622b36a48608813c2238cd1 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:09:15 +0100 Subject: [PATCH 041/176] Correzione chiamata alla classe di build. --- scripts/loaih-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/loaih-build b/scripts/loaih-build index 8392b37..fb03a43 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -23,7 +23,7 @@ def build(arch, language, offline, portable, updatable, download, storage, check arches = [ arch.lower() ] for arch in arches: - obj = loaih.build(query, arch) + obj = loaih.Build(query, arch) obj.language = language obj.offline_help = offlinehelp From 3557393d385cf4f0ab9c7bba394db9bb9d2968e3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:18:09 +0100 Subject: [PATCH 042/176] Chiamata sbagliata su appimagefilename. --- loaih/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index 76afa7b..ee0e7c5 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -20,6 +20,7 @@ class Build(object): self.updatable = True self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' + self.appimagefilename = {} # Getting versions and so on v = versions.BuildVersion(self.query) From da9d9807feea2793f59d8b241437558dfd08853b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:23:46 +0100 Subject: [PATCH 043/176] Correzione firma di funzione non corretta. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index ee0e7c5..6d0c812 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -127,7 +127,7 @@ class Build(object): # Build the requested version. self.__unpackbuild__() - def __unpackbuild__(self, languageset = 'full', offlinehelp = False): + def __unpackbuild__(self): # We start by filtering out tarballs from the list buildtarballs = [ self.__tarballs__[0] ] From 60db034067a795eadda10464c948ecf73fedffa8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:25:13 +0100 Subject: [PATCH 044/176] Correzione chiamata in script di build. --- scripts/loaih-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/loaih-build b/scripts/loaih-build index fb03a43..0ce4031 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -26,7 +26,7 @@ def build(arch, language, offline, portable, updatable, download, storage, check obj = loaih.Build(query, arch) obj.language = language - obj.offline_help = offlinehelp + obj.offline_help = offline obj.portable = portable obj.updatable = updatable From c2d3c9d21aedcd611b7a9b9904ba2ac308b9e518 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:30:22 +0100 Subject: [PATCH 045/176] Correzione chiamata di variabile in controllo di esistenza build. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 6d0c812..0792e7d 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -51,7 +51,7 @@ class Build(object): for arch in self.arch: res = subprocess.check_output("find {path} -name '{appimage}'".format( path = self.storage_path, - appimage = self.appimagefilename[arch] + '.zsync' if self.updatable else '' + appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '') ), shell=True).decode('utf-8').strip('\n') if len(res) > 1: self.built = True From baa0564c1686d069bf7ce66aac240aee12696833 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:34:38 +0100 Subject: [PATCH 046/176] Correzione a check(). --- loaih/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 0792e7d..795b6df 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -43,10 +43,7 @@ class Build(object): def check(self, storage_path): """Checking if the requested AppImage has been already built.""" - self.storage_path = storage_path - - if self.portable: - self.storage_path += "/portable" + self.storage_path = storage_path + ('/portable' if self.portable else '') for arch in self.arch: res = subprocess.check_output("find {path} -name '{appimage}'".format( @@ -187,7 +184,7 @@ class Build(object): # Building app if self.updatable: zsync = self.appimagefilename[arch] + '.zsync' - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = zsync, appname = self.__appname__), shell=True) + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) else: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.appname), shell=True) From 5745759488bd483afc73107ea738ad4bff81aac9 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:38:07 +0100 Subject: [PATCH 047/176] Altre correzioni alle chiamate. --- loaih/__init__.py | 8 +------- scripts/loaih-build | 5 ++++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 795b6df..522f114 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -208,16 +208,10 @@ class Build(object): subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) - def move(self, outdir): + def move(self): """Moves built versions to definitive storage.""" if self.built: return - if not outdir in self.storage_path: - # Let's create an updated storage_path - if self.portable: - self.storage_path = outdir + '/portable' - else: - self.storage_path = outdir os.chdir(self.appnamedir) subprocess.run("find . -iname '*.AppImage*' -exec cp {} %s \;" % self.storage_path, shell=True) diff --git a/scripts/loaih-build b/scripts/loaih-build index 0ce4031..dfbc13a 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -32,10 +32,13 @@ def build(arch, language, offline, portable, updatable, download, storage, check if check: obj.check(storage) + else: + obj.storage_path = storage + obj.download(download) obj.build() obj.checksums() - obj.move(storage) + obj.move() del obj if __name__ == '__main__': From d14a64fa8b95dd2edff5772cbee0e4f63ccc2325 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:40:24 +0100 Subject: [PATCH 048/176] Altre correzioni a appimagefilename. --- loaih/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 522f114..fd96692 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -30,8 +30,8 @@ class Build(object): if ',' in self.language: languagepart += self.language.replace(',', '-') - self.appimagefilename['x86'] = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + '-' + v.version + languagepart + '.help' if self.offline_help else '' + '-x86.AppImage' - self.appimagefilename['x86_64'] = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + '-' + v.version + languagepart + '.help' if self.offline_help else '' + '-x86_64.AppImage' + self.appimagefilename['x86'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' + self.appimagefilename['x86_64'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' # Creating a tempfile self.builddir = tempfile.mkdtemp() From 960df20639b0b4cc3e3fb45b79a2caf83e95daa3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:42:53 +0100 Subject: [PATCH 049/176] Probabile problema con valori di array. --- loaih/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index fd96692..eec2d54 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -30,8 +30,8 @@ class Build(object): if ',' in self.language: languagepart += self.language.replace(',', '-') - self.appimagefilename['x86'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' - self.appimagefilename['x86_64'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' + self.appimagefilename[u'x86'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' # Creating a tempfile self.builddir = tempfile.mkdtemp() From 8eaa14d4d382d20a48461e0a79432aba9355ebb5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:44:30 +0100 Subject: [PATCH 050/176] Debug di appimage. --- loaih/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index eec2d54..34199d5 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -46,6 +46,7 @@ class Build(object): self.storage_path = storage_path + ('/portable' if self.portable else '') for arch in self.arch: + print("DEBUG: {appimage}".format(appimage = self.appimagefilename[arch])) res = subprocess.check_output("find {path} -name '{appimage}'".format( path = self.storage_path, appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '') From 9ae9585d20624a6a52616eee0488d5bba8a474fb Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:46:07 +0100 Subject: [PATCH 051/176] Debug di appimagefilename. --- loaih/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index 34199d5..180c5d7 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -33,6 +33,8 @@ class Build(object): self.appimagefilename[u'x86'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' self.appimagefilename[u'x86_64'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' + print("DEBUG:") + print(self.appimagefilename) # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = [] From daff0535d3d03fdc458fc76dc5ee5dd038e08c2e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:48:42 +0100 Subject: [PATCH 052/176] Trovato forse problema: passaggio di arch. --- loaih/__init__.py | 4 ++-- scripts/loaih-build | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 180c5d7..f6efe2d 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -29,12 +29,12 @@ class Build(object): languagepart = "." if ',' in self.language: languagepart += self.language.replace(',', '-') + else: + languagepart += self.language self.appimagefilename[u'x86'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' self.appimagefilename[u'x86_64'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' - print("DEBUG:") - print(self.appimagefilename) # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = [] diff --git a/scripts/loaih-build b/scripts/loaih-build index dfbc13a..2e9bcf6 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -18,7 +18,7 @@ def build(arch, language, offline, portable, updatable, download, storage, check # Parsing options if arch.lower() == 'all': # We need to build it twice. - arches = [ 'x86', 'x86_64' ] + arches = [ u'x86', u'x86_64' ] else: arches = [ arch.lower() ] From 0df231e4509e6eae9a7d3bc746ecf53f4848633a Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 00:54:44 +0100 Subject: [PATCH 053/176] Trovato forse problema: passaggio di arch. --- loaih/__init__.py | 5 +---- scripts/loaih-build | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index f6efe2d..2c1c1f2 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -49,10 +49,7 @@ class Build(object): for arch in self.arch: print("DEBUG: {appimage}".format(appimage = self.appimagefilename[arch])) - res = subprocess.check_output("find {path} -name '{appimage}'".format( - path = self.storage_path, - appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '') - ), shell=True).decode('utf-8').strip('\n') + res = subprocess.check_output("find {path} -name '{appimage}'".format(path = self.storage_path, appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '')), shell=True).decode('utf-8').strip('\n') if len(res) > 1: self.built = True diff --git a/scripts/loaih-build b/scripts/loaih-build index 2e9bcf6..111e64f 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -20,26 +20,25 @@ def build(arch, language, offline, portable, updatable, download, storage, check # We need to build it twice. arches = [ u'x86', u'x86_64' ] else: - arches = [ arch.lower() ] + arches = [ arch.lower().decode('utf-8') ] - for arch in arches: - obj = loaih.Build(query, arch) + obj = loaih.Build(query, arches) - obj.language = language - obj.offline_help = offline - obj.portable = portable - obj.updatable = updatable + obj.language = language + obj.offline_help = offline + obj.portable = portable + obj.updatable = updatable - if check: - obj.check(storage) - else: - obj.storage_path = storage + if check: + obj.check(storage) + else: + obj.storage_path = storage - obj.download(download) - obj.build() - obj.checksums() - obj.move() - del obj + obj.download(download) + obj.build() + obj.checksums() + obj.move() + del obj if __name__ == '__main__': build() From 3422c5e93138f0e1df3fa9cd036da6f0ece0fde1 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:10:17 +0100 Subject: [PATCH 054/176] Alcune pesanti correzioni ai meccanismi di download e alla raccolta dei tarball. --- loaih/__init__.py | 79 +++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 2c1c1f2..be281dc 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -13,7 +13,7 @@ class Build(object): """Build all versions that can be found in the indicated repo.""" self.query = query self.arch = arch - self.url = '' + self.url = {} self.language = 'basic' self.offline_help = False self.portable = False @@ -25,6 +25,14 @@ class Build(object): # Getting versions and so on v = versions.BuildVersion(self.query) + # Creating a tempfile + self.builddir = tempfile.mkdtemp() + self.tarballs = {} + self.appname = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + self.version = v.version + self.url = v.basedirurl + self.built = False + # Building expected AppImageName languagepart = "." if ',' in self.language: @@ -32,24 +40,19 @@ class Build(object): else: languagepart += self.language - self.appimagefilename[u'x86'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = ('LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev') + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' + self.appimagefilename[u'x86'] = self.appname + '-' + self.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' - # Creating a tempfile - self.builddir = tempfile.mkdtemp() - self.tarballs = [] - self.appname = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' - self.version = v.version - self.url = v.basedirurl - self.built = False def check(self, storage_path): """Checking if the requested AppImage has been already built.""" self.storage_path = storage_path + ('/portable' if self.portable else '') for arch in self.arch: - print("DEBUG: {appimage}".format(appimage = self.appimagefilename[arch])) - res = subprocess.check_output("find {path} -name '{appimage}'".format(path = self.storage_path, appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '')), shell=True).decode('utf-8').strip('\n') + res = subprocess.check_output("find {path} -name '{appimage}'".format( + path = self.storage_path, + appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '') + ), shell=True).decode('utf-8').strip('\n') if len(res) > 1: self.built = True @@ -65,25 +68,28 @@ class Build(object): # Let's explore the remote folder. self.download_path = download_path - for a in self.arch: - contents = etree.HTML(urllib.request.urlopen(self.url[a]).read()).xpath("//td/a") - self.tarballs = [ x.text for x in contents if x.text.endswith('tar.gz') ] - maintarball = self.tarballs[0] - mainarr = maintarball.split('') - self.appname = mainarr[0] - self.version = mainarr[1] + for arch in self.arch: + # Checking if a valid path has been provided + if self.url[arch] == '-': + print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) + continue - os.makedirs(self.downloaddir, existok = True) - os.chdir(self.downloaddir) - for archive in self.tarballs: + contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") + self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') ] + tarballs = self.tarballs[arch] + maintarball = tarballs[0] + + os.makedirs(self.download_path, existok = True) + os.chdir(self.download_path) + for archive in tarballs: # If the archive is already there, do not do anything. - if os.path.exists(os.path.join(self.downloaddir, archive)): + if os.path.exists(os.path.join(self.download_path, archive)): print("Archive %s is already there! Sweet" % archive) continue # Download the archive try: - urllib.request.urlretrieve(self.url + archive, archive) + urllib.request.urlretrieve(self.url[arch] + archive, archive) except: print("Failed to download {archive}.".format(archive = archive)) @@ -110,6 +116,11 @@ class Build(object): self.updatable = False for arch in self.arch: + # Checking if a valid path has been provided + if self.url[arch] == '-': + # User has been warned in download step. + continue + # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') @@ -122,31 +133,31 @@ class Build(object): os.chmod('appimagetool', 0o755) # Build the requested version. - self.__unpackbuild__() + self.__unpackbuild__(arch) - def __unpackbuild__(self): + def __unpackbuild__(self, arch): # We start by filtering out tarballs from the list - buildtarballs = [ self.__tarballs__[0] ] + buildtarballs = [ self.tarballs[arch][0] ] # Let's process standard languages and append results to the # buildtarball if self.language == 'basic': - buildtarballs.extend([ x for x in self.tarballs if 'langpack_en-GB' in x]) + buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs if 'helppack_en-GB' in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if 'helppack_en-GB' in x ]) if self.language == 'standard': for lang in Build.LANGSTD: - buildtarballs.extend([ x for x in self.tarballs if ('langpack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) if self.offlinehelp: - buildtarballs.extend([ x for x in self.tarballs if ('helppack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if ('helppack' + lang) in x ]) else: # In any other cases, we build with all languages if not self.offlinehelp: - buildtarballs.extend([ x for x in self.tarballs if 'langpack' in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) else: # We need also all help. Let's replace buildtarball with the # whole bunch - buildtarballs = self.tarballs + buildtarballs = self.tarballs[arch] # Unpacking the tarballs for archive in buildtarballs: @@ -177,7 +188,7 @@ class Build(object): os.chmod(dest, 0o755) # Setting app version - appversion = self.version + '.' + self.language if not ',' in self.language else self.language.replace(',', '-') + appversion = self.version + '.' + (self.language if not ',' in self.language else self.language.replace(',', '-')) if self.offline_help: appversion += '.help' From 71edb785d2ab168bb81fed70f876e35d1eb8988c Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:11:17 +0100 Subject: [PATCH 055/176] Correzione opzione exist_ok su makedirs() --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index be281dc..4052994 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -79,7 +79,7 @@ class Build(object): tarballs = self.tarballs[arch] maintarball = tarballs[0] - os.makedirs(self.download_path, existok = True) + os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) for archive in tarballs: # If the archive is already there, do not do anything. From e44a70442fe731c442f97d6fda182e9f4a6e3f8c Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:14:28 +0100 Subject: [PATCH 056/176] Correzione flag offline_help. --- loaih/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 4052994..8eccb8f 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -148,11 +148,11 @@ class Build(object): if self.language == 'standard': for lang in Build.LANGSTD: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - if self.offlinehelp: + if self.offline_help: buildtarballs.extend([ x for x in self.tarballs[arch] if ('helppack' + lang) in x ]) else: # In any other cases, we build with all languages - if not self.offlinehelp: + if not self.offline_help: buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) else: # We need also all help. Let's replace buildtarball with the From 1c46f9846f31fb9a62fa3bec0e398bcc5e58ce93 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:18:37 +0100 Subject: [PATCH 057/176] Sistemato appname. --- loaih/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 8eccb8f..a33726f 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -169,8 +169,7 @@ class Build(object): subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) - appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True, cwd=self.appimagedir) + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) From ae974e613aea69c002f3d94d56cb717308ddb7a8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:29:27 +0100 Subject: [PATCH 058/176] Supporto portable. --- loaih/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index a33726f..d682d94 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -167,6 +167,10 @@ class Build(object): os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + if self.portable: + shortversion = self.version.split('.')[0-2].join('.') + subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/{version}|g' {} \+".format(version = shortversion), shell=True, cwd=self.appimagedir) + # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) From be24ebf75e2b2a59d99c6300f4f3b49a26c875c8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:33:47 +0100 Subject: [PATCH 059/176] Ulteriori correzioni per portable. --- loaih/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index d682d94..32643cc 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -47,6 +47,11 @@ class Build(object): def check(self, storage_path): """Checking if the requested AppImage has been already built.""" self.storage_path = storage_path + ('/portable' if self.portable else '') + # Incompatibilities - if portable and updatable are asked together, + # only portable will be built. + if self.portable and self.updatable: + print("Upgradable and portable options were required together. Building only portable.") + self.updatable = False for arch in self.arch: res = subprocess.check_output("find {path} -name '{appimage}'".format( @@ -109,12 +114,6 @@ class Build(object): if self.portable and not 'portable' in self.storage_path: self.storage_path += "/portable" - # Incompatibilities - if portable and updatable are asked together, - # only portable will be built. - if self.portable and self.updatable: - print("Upgradable and portable options were required together. Building only portable.") - self.updatable = False - for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': From 5430000a32816be8c78f1c9fdf1d0ae8dfd00a2a Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:37:03 +0100 Subject: [PATCH 060/176] Correzione shortversion per portable. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 32643cc..ecd087c 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -167,7 +167,7 @@ class Build(object): # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) if self.portable: - shortversion = self.version.split('.')[0-2].join('.') + shortversion = str.join('.', self.version.split('.')[:3]) subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/{version}|g' {} \+".format(version = shortversion), shell=True, cwd=self.appimagedir) # Changing desktop file From d1a2b5046180457e272ef3ea36f7ac1893a06d48 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Mon, 21 Mar 2022 01:39:56 +0100 Subject: [PATCH 061/176] Correzione sostituzione della shortversion nella stringa di replacement. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index ecd087c..5c6398e 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -168,7 +168,7 @@ class Build(object): subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) if self.portable: shortversion = str.join('.', self.version.split('.')[:3]) - subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/{version}|g' {} \+".format(version = shortversion), shell=True, cwd=self.appimagedir) + subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) From 66bcd2b39cdf5e2dbf9768d067b3c659dca82f07 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Tue, 22 Mar 2022 01:24:55 +0100 Subject: [PATCH 062/176] Supporto a daily e prerelease. --- loaih/__init__.py | 23 +++++++++++++++-------- loaih/versions.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 5c6398e..fd94008 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -28,24 +28,26 @@ class Build(object): # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} - self.appname = 'LibreOffice' if not self.query == 'daily' else 'LibreOfficeDev' + self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' self.version = v.version self.url = v.basedirurl self.built = False # Building expected AppImageName - languagepart = "." + self.languagepart = "." if ',' in self.language: - languagepart += self.language.replace(',', '-') + self.languagepart += self.language.replace(',', '-') else: - languagepart += self.language + self.languagepart += self.language - self.appimagefilename[u'x86'] = self.appname + '-' + self.version + languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' + self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' def check(self, storage_path): """Checking if the requested AppImage has been already built.""" + self.storage_path += ('/daily' if self.query == 'daily' else '') + self.storage_path += ('/prerelease' if self.query == 'prerelease' else '') self.storage_path = storage_path + ('/portable' if self.portable else '') # Incompatibilities - if portable and updatable are asked together, # only portable will be built. @@ -196,8 +198,13 @@ class Build(object): # Building app if self.updatable: - zsync = self.appimagefilename[arch] + '.zsync' - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + # for the daily build, updatable builds work if the latest one + # provide updates. + if self.query == 'daily': + zsync = self.appname + self.version.split('-')[0] + self.languagepart + ('.help' if self.offline_help else '') + '-' + arch + '.AppImage.zsync' + else: + zsync = self.appimagefilename[arch] + '.zsync' + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = zsync, appname = self.appname), shell=True) else: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.appname), shell=True) diff --git a/loaih/versions.py b/loaih/versions.py index 7def95d..fa0d972 100644 --- a/loaih/versions.py +++ b/loaih/versions.py @@ -8,7 +8,8 @@ from packaging.version import parse as parse_version class BuildVersion(object): ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" - DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/current/" + DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" + PRERELEASE = "https://dev-builds.libreoffice.org/pre-releases/deb/x86_64/" def __init__(self, query): self.query = query @@ -45,17 +46,33 @@ class BuildVersion(object): basedirurl = {} version = '' if branch == 'daily': - basedirurl = { 'x86_64': BuildVersion.DAILY, 'x86': '-' } - version = etree.HTML(urllib.request.urlopen(BuildVersion.DAILY).read()).xpath('//td/a')[1].text.split('_')[1] + # The daily builds can be mostly distinguished by the day of build + # (official version is constant. + + # The last built version is the next-to-last version [-2] on the page. + fulldailypath = etree.HTML(urllib.request.urlopen(BuildVersion.DAILY).read()).xpath('//td/a')[-2].text + dailyversion = fulldailypath.split('_')[0].replace('-', '') + version + newurl = str.join('/', [ BuildVersion.DAILY, fulldailypath, '' ]) + + basedirurl = { u'x86_64': newurl, u'x86': '-' } + version = etree.HTML(urllib.request.urlopen(newurl).read()).xpath('//td/a')[1].text.split('_')[1] + + return { 'version': version + '-' + dailyversion, 'basedirurl': basedirurl } + + if branch == 'prerelease': + version = etree.HTML(urllib.request.urlopen(BuildVersion.PRERELEASE).read()).xpath('//td/a')[1].text.split('_')[1] + basedirurl = { u'x86': '-', u'x86_64': BuildVersion.PRERELEASE } return { 'version': version, 'basedirurl': basedirurl } + # Stable releases. versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') index = 1 if branch == 'still': - index = 2 + index = -2 elif branch == 'fresh': - index = 3 + index = -1 version = self.__getlatestrel(versions[index].text.strip('/')) return { 'version': version, 'basedirurl': self.__getbaseurl(version) } @@ -67,10 +84,10 @@ class BuildVersion(object): # x86 binaries are not anymore offered after 6.3.0. if parse_version(version) < parse_version('6.3.0'): - basedirurl['x86'] = url + 'x86/' + basedirurl[u'x86'] = url + 'x86/' else: - basedirurl['x86'] = '-' + basedirurl[u'x86'] = '-' - basedirurl['x86_64'] = url + 'x86_64/' + basedirurl[u'x86_64'] = url + 'x86_64/' return basedirurl From 181940b365133ca48a1556b227cc4d6266be3b40 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Tue, 22 Mar 2022 02:04:25 +0100 Subject: [PATCH 063/176] Corretta determinazione storage in cui spostare le build. --- loaih/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index fd94008..2662ec3 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -46,9 +46,10 @@ class Build(object): def check(self, storage_path): """Checking if the requested AppImage has been already built.""" + self.storage_path = storage_path self.storage_path += ('/daily' if self.query == 'daily' else '') self.storage_path += ('/prerelease' if self.query == 'prerelease' else '') - self.storage_path = storage_path + ('/portable' if self.portable else '') + self.storage_path += ('/portable' if self.portable else '') # Incompatibilities - if portable and updatable are asked together, # only portable will be built. if self.portable and self.updatable: @@ -201,9 +202,8 @@ class Build(object): # for the daily build, updatable builds work if the latest one # provide updates. if self.query == 'daily': - zsync = self.appname + self.version.split('-')[0] + self.languagepart + ('.help' if self.offline_help else '') + '-' + arch + '.AppImage.zsync' - else: - zsync = self.appimagefilename[arch] + '.zsync' + appversion = 'daily' + zsync = self.appimagefilename[arch] + '.zsync' subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = zsync, appname = self.appname), shell=True) else: From 2410098953f2336a240199a67f5a9cd2c4b7ac61 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Tue, 22 Mar 2022 02:10:09 +0100 Subject: [PATCH 064/176] Aggiustato build per daily. --- loaih/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 2662ec3..553816f 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -83,7 +83,7 @@ class Build(object): continue contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') ] + self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz' and 'deb' in x.text) ] tarballs = self.tarballs[arch] maintarball = tarballs[0] @@ -201,8 +201,6 @@ class Build(object): if self.updatable: # for the daily build, updatable builds work if the latest one # provide updates. - if self.query == 'daily': - appversion = 'daily' zsync = self.appimagefilename[arch] + '.zsync' subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = zsync, appname = self.appname), shell=True) From 425adb5545e998bc7abfb75a6cd4f4ab7f9718e3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 01:32:15 +0100 Subject: [PATCH 065/176] Corretta logica di build con il discorso updatable: solo se richiesta versione generica o daily. --- loaih/__init__.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 553816f..a1f4d1b 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -12,6 +12,7 @@ class Build(object): def __init__(self, query, arch): """Build all versions that can be found in the indicated repo.""" self.query = query + self.queried_name = False if '.' in self.query else True self.arch = arch self.url = {} self.language = 'basic' @@ -21,6 +22,7 @@ class Build(object): self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' self.appimagefilename = {} + self.zsyncfilename = {} # Getting versions and so on v = versions.BuildVersion(self.query) @@ -40,8 +42,14 @@ class Build(object): else: self.languagepart += self.language - self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + ('.help' if self.offline_help else '') + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + ('.help' if self.offline_help else '') + '-x86_64.AppImage' + self.helppart = '.help' if self.offline_help else '' + + self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + self.helppart + '-x86_64.AppImage' + + version = str.join('.', self.version.split('.')[0-2]) + self.zsyncfilename[u'x86'] = self.appname + '-' + version + self.languagepart + self.helppart + '-x86.AppImage.zsync' + self.zsyncfilename[u'x86_64'] = self.appname + '-' + version + self.languagepart + self.helppart + '-x86_64.AppImage.zsync' def check(self, storage_path): @@ -56,10 +64,15 @@ class Build(object): print("Upgradable and portable options were required together. Building only portable.") self.updatable = False + if self.updatable and not self.queried_name: + # If the queried version was a numbered version, doesn't make sense + # to build an updatable version. + self.updatable = False + for arch in self.arch: res = subprocess.check_output("find {path} -name '{appimage}'".format( path = self.storage_path, - appimage = self.appimagefilename[arch] + ('.zsync' if self.updatable else '') + appimage = self.appimagefilename[arch] ), shell=True).decode('utf-8').strip('\n') if len(res) > 1: self.built = True @@ -166,9 +179,11 @@ class Build(object): subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) os.chdir(self.appnamedir) + os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + if self.portable: shortversion = str.join('.', self.version.split('.')[:3]) subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) @@ -198,11 +213,12 @@ class Build(object): appversion += '.help' # Building app - if self.updatable: - # for the daily build, updatable builds work if the latest one - # provide updates. - zsync = self.appimagefilename[arch] + '.zsync' - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = zsync, appname = self.appname), shell=True) + if self.updatable and self.queried_name: + # Updatable make sense only for generic images for fresh, still, + # daily. If a request was for a specific version, I'd not build an + # updatable version. + # zsync name was generated already + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = self.zsyncfilename[arch], appname = self.appname), shell=True) else: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.appname), shell=True) From 5bb82074093a74bbaf09953b19c5b0229b13ea81 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 01:35:06 +0100 Subject: [PATCH 066/176] Corretta sintassi per selezionare solo deb. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index a1f4d1b..c00b62c 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -96,7 +96,7 @@ class Build(object): continue contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz' and 'deb' in x.text) ] + self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] tarballs = self.tarballs[arch] maintarball = tarballs[0] From 7afedb2b906815ac0ff70e0397fdcc2ff3f46036 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 01:45:29 +0100 Subject: [PATCH 067/176] Corretto piccolo errore di sintassi. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index c00b62c..b07746c 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -47,7 +47,7 @@ class Build(object): self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + self.helppart + '-x86_64.AppImage' - version = str.join('.', self.version.split('.')[0-2]) + version = str.join('.', self.version.split('.')[0:2]) self.zsyncfilename[u'x86'] = self.appname + '-' + version + self.languagepart + self.helppart + '-x86.AppImage.zsync' self.zsyncfilename[u'x86_64'] = self.appname + '-' + version + self.languagepart + self.helppart + '-x86_64.AppImage.zsync' From f48189e3b33517c5d0ca627ac5c653b09798b3b8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 01:46:08 +0100 Subject: [PATCH 068/176] Correzione versionamento della appimage con zsync. --- loaih/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index b07746c..b055959 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -47,9 +47,9 @@ class Build(object): self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + self.helppart + '-x86_64.AppImage' - version = str.join('.', self.version.split('.')[0:2]) - self.zsyncfilename[u'x86'] = self.appname + '-' + version + self.languagepart + self.helppart + '-x86.AppImage.zsync' - self.zsyncfilename[u'x86_64'] = self.appname + '-' + version + self.languagepart + self.helppart + '-x86_64.AppImage.zsync' + myver = str.join('.', self.version.split('.')[0:2]) + self.zsyncfilename[u'x86'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86.AppImage.zsync' + self.zsyncfilename[u'x86_64'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86_64.AppImage.zsync' def check(self, storage_path): From 17fb8bb2b2534465a7b4cd072d293159eb59d6a3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:04:02 +0100 Subject: [PATCH 069/176] Rivista logica di build per query nominali. --- loaih/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index b055959..725e47d 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -22,7 +22,6 @@ class Build(object): self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' self.appimagefilename = {} - self.zsyncfilename = {} # Getting versions and so on v = versions.BuildVersion(self.query) @@ -44,12 +43,18 @@ class Build(object): self.helppart = '.help' if self.offline_help else '' - self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + self.helppart + '-x86_64.AppImage' + # If the build was called by queried name, build from latest release available but build with the most generic name + if self.queried_name: + myver = str.join('.', self.version.split('.')[0:2]) + + self.appimagefilename[u'x86'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86_64.AppImage' + + else: + self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + self.helppart + '-x86_64.AppImage' + - myver = str.join('.', self.version.split('.')[0:2]) - self.zsyncfilename[u'x86'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86.AppImage.zsync' - self.zsyncfilename[u'x86_64'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86_64.AppImage.zsync' def check(self, storage_path): @@ -218,7 +223,7 @@ class Build(object): # daily. If a request was for a specific version, I'd not build an # updatable version. # zsync name was generated already - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = self.zsyncfilename[arch], appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) else: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.appname), shell=True) From 8c298e85dede887d06217561c7e0352291808236 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:10:25 +0100 Subject: [PATCH 070/176] Ancora cambio logica build nominali. --- loaih/__init__.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 725e47d..a734ebe 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -21,6 +21,7 @@ class Build(object): self.updatable = True self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' + self.appversion = '' self.appimagefilename = {} # Getting versions and so on @@ -46,15 +47,12 @@ class Build(object): # If the build was called by queried name, build from latest release available but build with the most generic name if self.queried_name: myver = str.join('.', self.version.split('.')[0:2]) - - self.appimagefilename[u'x86'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + myver + self.languagepart + self.helppart + '-x86_64.AppImage' - + self.appversion = myver + self.languagepart + self.helppart else: - self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + v.version + self.languagepart + self.helppart + '-x86_64.AppImage' - + self.appversion = self.version + self.languagepart + self.helppart + self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86_64.AppImage' def check(self, storage_path): @@ -212,23 +210,18 @@ class Build(object): urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) - # Setting app version - appversion = self.version + '.' + (self.language if not ',' in self.language else self.language.replace(',', '-')) - if self.offline_help: - appversion += '.help' - # Building app if self.updatable and self.queried_name: # Updatable make sense only for generic images for fresh, still, # daily. If a request was for a specific version, I'd not build an # updatable version. # zsync name was generated already - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) else: - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) - print("Built AppImage version {version}".format(version = appversion)) + print("Built AppImage version {version}".format(version = self.appversion)) # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): From da8adc785fa1e14b53a1d51e0b0a46be3e843883 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:32:02 +0100 Subject: [PATCH 071/176] Cambiata la logica di verifica della versione. Ogni versione nominale viene buildata due volte, una con nome generico e una con specifico con versione completa. --- loaih/__init__.py | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index a734ebe..ee99036 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -23,6 +23,8 @@ class Build(object): self.download_path = '/var/tmp/downloads' self.appversion = '' self.appimagefilename = {} + self.genappversion = '' + self.genappimagefilename = {} # Getting versions and so on v = versions.BuildVersion(self.query) @@ -45,12 +47,12 @@ class Build(object): self.helppart = '.help' if self.offline_help else '' # If the build was called by queried name, build from latest release available but build with the most generic name - if self.queried_name: - myver = str.join('.', self.version.split('.')[0:2]) - self.appversion = myver + self.languagepart + self.helppart - else: - self.appversion = self.version + self.languagepart + self.helppart + myver = str.join('.', self.version.split('.')[0:2]) + self.genappversion = myver + self.languagepart + self.helppart + self.genappimagefilename[u'x86'] = self.appname + '-' + self.genappversion + self.languagepart + self.helppart + '-x86.AppImage' + self.genappimagefilename[u'x86_64'] = self.appname + '-' + self.genappversion + self.languagepart + self.helppart + '-x86_64.AppImage' + self.appversion = self.version + self.languagepart + self.helppart self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' self.appimagefilename[u'x86_64'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86_64.AppImage' @@ -73,6 +75,16 @@ class Build(object): self.updatable = False for arch in self.arch: + # For generalized builds, we need to check if there are .ver file + # and it contains the specific version found. + res = subprocess.check_output("find {path} -name {appimage}'".format( + path = self.storage_path, + appimage = self.genappimagefilename[arch] + '.ver' + ), shell=True).decode('utf-8').strip('\n') + for file in res: + if self.version in open(file, 'r').read(): + self.built = True + res = subprocess.check_output("find {path} -name '{appimage}'".format( path = self.storage_path, appimage = self.appimagefilename[arch] @@ -151,9 +163,11 @@ class Build(object): os.chmod('appimagetool', 0o755) # Build the requested version. + if self.queried_name: + self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) - def __unpackbuild__(self, arch): + def __unpackbuild__(self, arch, generalize = False): # We start by filtering out tarballs from the list buildtarballs = [ self.tarballs[arch][0] ] @@ -211,15 +225,28 @@ class Build(object): os.chmod(dest, 0o755) # Building app - if self.updatable and self.queried_name: + if self.updatable: # Updatable make sense only for generic images for fresh, still, # daily. If a request was for a specific version, I'd not build an # updatable version. # zsync name was generated already - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + + # If asked to do a generalized build: + if generalize: + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + # Build version file management + with open(self.appimagefilename[arch] + '.ver', 'w') as v: + v.write(self.version) + else: + subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) else: - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) + if generalize: + subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.genappversion, appname = self.appname), shell=True) + with open(self.appimagefilename[arch] + '.ver', 'w') as v: + v.write(self.version) + else: + subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) print("Built AppImage version {version}".format(version = self.appversion)) @@ -246,7 +273,7 @@ class Build(object): return os.chdir(self.appnamedir) - subprocess.run("find . -iname '*.AppImage*' -exec cp {} %s \;" % self.storage_path, shell=True) + subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.storage_path, shell=True) def __del__(self): """Destructor""" From db419bb82596ad0e92bf9e60d8a56b6a963a9dda Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:42:53 +0100 Subject: [PATCH 072/176] Cambio logica codice per trovare file buildati. --- loaih/__init__.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index ee99036..7519b15 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -49,12 +49,12 @@ class Build(object): # If the build was called by queried name, build from latest release available but build with the most generic name myver = str.join('.', self.version.split('.')[0:2]) self.genappversion = myver + self.languagepart + self.helppart - self.genappimagefilename[u'x86'] = self.appname + '-' + self.genappversion + self.languagepart + self.helppart + '-x86.AppImage' - self.genappimagefilename[u'x86_64'] = self.appname + '-' + self.genappversion + self.languagepart + self.helppart + '-x86_64.AppImage' + self.genappimagefilename[u'x86'] = self.appname + '-' + self.genappversion + '-x86.AppImage' + self.genappimagefilename[u'x86_64'] = self.appname + '-' + self.genappversion + '-x86_64.AppImage' self.appversion = self.version + self.languagepart + self.helppart - self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86_64.AppImage' + self.appimagefilename[u'x86'] = self.appname + '-' + self.appversion + '-x86.AppImage' + self.appimagefilename[u'x86_64'] = self.appname + '-' + self.appversion + '-x86_64.AppImage' def check(self, storage_path): @@ -77,20 +77,23 @@ class Build(object): for arch in self.arch: # For generalized builds, we need to check if there are .ver file # and it contains the specific version found. - res = subprocess.check_output("find {path} -name {appimage}'".format( + res = subprocess.run("find {path} -name {appimage}'".format( path = self.storage_path, appimage = self.genappimagefilename[arch] + '.ver' - ), shell=True).decode('utf-8').strip('\n') - for file in res: - if self.version in open(file, 'r').read(): - self.built = True + ), shell=True, capture_output=True) + if res.stdout: + # All good, the command was executed fine. + for file in res.stdout.decode('utf-8').strip('\n').split('\n'): + if self.version in open(file, 'r').read(): + self.built = True - res = subprocess.check_output("find {path} -name '{appimage}'".format( + res = subprocess.run("find {path} -name '{appimage}'".format( path = self.storage_path, appimage = self.appimagefilename[arch] - ), shell=True).decode('utf-8').strip('\n') - if len(res) > 1: - self.built = True + ), shell=True, capture_output=True) + if res.stdout: + if len(res.stdout.decode('utf-8').strip('\n')) > 1: + self.built = True if self.built: print("The requested AppImage already exists on storage. I'll skip downloading, building and moving the results.") From 960bab4dee62067dcec983f5b3fb2a15f1bad6fd Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:45:01 +0100 Subject: [PATCH 073/176] Expliciting which file is checking out. --- loaih/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index 7519b15..a00384f 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -77,6 +77,7 @@ class Build(object): for arch in self.arch: # For generalized builds, we need to check if there are .ver file # and it contains the specific version found. + print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] . 'ver')) res = subprocess.run("find {path} -name {appimage}'".format( path = self.storage_path, appimage = self.genappimagefilename[arch] + '.ver' @@ -87,6 +88,7 @@ class Build(object): if self.version in open(file, 'r').read(): self.built = True + print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) res = subprocess.run("find {path} -name '{appimage}'".format( path = self.storage_path, appimage = self.appimagefilename[arch] From 24de2a2d2192c763527d3959649bd1e34869eb6c Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:45:55 +0100 Subject: [PATCH 074/176] Correzione piccolo errore di sintassi. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index a00384f..f8806af 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -77,7 +77,7 @@ class Build(object): for arch in self.arch: # For generalized builds, we need to check if there are .ver file # and it contains the specific version found. - print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] . 'ver')) + print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) res = subprocess.run("find {path} -name {appimage}'".format( path = self.storage_path, appimage = self.genappimagefilename[arch] + '.ver' From 6fe41c6e765eec90fe4a1f3293de0477426cd188 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 26 Mar 2022 02:54:06 +0100 Subject: [PATCH 075/176] Spostamento codice per calcolo versione. --- loaih/__init__.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index f8806af..fbcfdf0 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -37,6 +37,23 @@ class Build(object): self.url = v.basedirurl self.built = False + def check(self, storage_path): + """Checking if the requested AppImage has been already built.""" + self.storage_path = storage_path + self.storage_path += ('/daily' if self.query == 'daily' else '') + self.storage_path += ('/prerelease' if self.query == 'prerelease' else '') + self.storage_path += ('/portable' if self.portable else '') + # Incompatibilities - if portable and updatable are asked together, + # only portable will be built. + if self.portable and self.updatable: + print("Upgradable and portable options were required together. Building only portable.") + self.updatable = False + + if self.updatable and not self.queried_name: + # If the queried version was a numbered version, doesn't make sense + # to build an updatable version. + self.updatable = False + # Building expected AppImageName self.languagepart = "." if ',' in self.language: @@ -56,24 +73,6 @@ class Build(object): self.appimagefilename[u'x86'] = self.appname + '-' + self.appversion + '-x86.AppImage' self.appimagefilename[u'x86_64'] = self.appname + '-' + self.appversion + '-x86_64.AppImage' - - def check(self, storage_path): - """Checking if the requested AppImage has been already built.""" - self.storage_path = storage_path - self.storage_path += ('/daily' if self.query == 'daily' else '') - self.storage_path += ('/prerelease' if self.query == 'prerelease' else '') - self.storage_path += ('/portable' if self.portable else '') - # Incompatibilities - if portable and updatable are asked together, - # only portable will be built. - if self.portable and self.updatable: - print("Upgradable and portable options were required together. Building only portable.") - self.updatable = False - - if self.updatable and not self.queried_name: - # If the queried version was a numbered version, doesn't make sense - # to build an updatable version. - self.updatable = False - for arch in self.arch: # For generalized builds, we need to check if there are .ver file # and it contains the specific version found. From 8d8f2b6dc446d1e5c1e646f08d10080c53b4f188 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 00:07:43 +0200 Subject: [PATCH 076/176] Alcuni miglioramenti di ricerca delle build in path specifici. --- loaih/__init__.py | 92 +++++++++++++++++++++++++++++++-------------- scripts/loaih-build | 2 +- setup.py | 2 +- 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index fbcfdf0..55566b9 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -3,7 +3,7 @@ import urllib.request import loaih.versions as versions from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil +import tempfile, os, sys, glob, subprocess, shutil, re class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] @@ -35,14 +35,21 @@ class Build(object): self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' self.version = v.version self.url = v.basedirurl - self.built = False + self.built = { u'x86': False, u'x86_64': False } + + # Preparing the default for the relative path on the storage for + # different versions. + # The path will evaluated as part of the check() function, as it is + # understood the storage_path can be changed before that phase. + self.relative_path = [] + self.full_path = '' def check(self, storage_path): """Checking if the requested AppImage has been already built.""" - self.storage_path = storage_path - self.storage_path += ('/daily' if self.query == 'daily' else '') - self.storage_path += ('/prerelease' if self.query == 'prerelease' else '') - self.storage_path += ('/portable' if self.portable else '') + # Mandate to the private function to calculate the full_path available + # for the storage and the checks. + self.__calculate_full_path__() + # Incompatibilities - if portable and updatable are asked together, # only portable will be built. if self.portable and self.updatable: @@ -78,32 +85,53 @@ class Build(object): # and it contains the specific version found. print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) res = subprocess.run("find {path} -name {appimage}'".format( - path = self.storage_path, + path = self.full_path, appimage = self.genappimagefilename[arch] + '.ver' - ), shell=True, capture_output=True) + ), shell=True, capture_output=True, env={ "LC_ALL": "C" }) + + if "No such file or directory" in res.stderr.decode('utf-8'): + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + if res.stdout: # All good, the command was executed fine. for file in res.stdout.decode('utf-8').strip('\n').split('\n'): if self.version in open(file, 'r').read(): - self.built = True + self.built[arch] = True print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) res = subprocess.run("find {path} -name '{appimage}'".format( - path = self.storage_path, + path = self.full_path, appimage = self.appimagefilename[arch] ), shell=True, capture_output=True) if res.stdout: if len(res.stdout.decode('utf-8').strip('\n')) > 1: - self.built = True + self.built[arch] = True - if self.built: - print("The requested AppImage already exists on storage. I'll skip downloading, building and moving the results.") + if self.built[arch]: + print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) + + def __calculate_full_path__(self): + """Calculate relative path of the build, based on internal other variables.""" + if len(self.relative_path) == 0: + if self.query == 'daily': + self.relative_path.append('daily') + elif self.query == 'prerelease': + self.relative_path.append('prerelease') + + # Not the same check, an additional one + if self.portable: + self.relative_path.append('portable') + + fullpath_arr = self.storage_path.split('/') + # Joining relative path only if it is not null + if len(self.relative_path) > 0: + fullpath_arr.expand(self.relative_path) + self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) def download(self, download_path): """Downloads the contents of the URL as it was a folder.""" - if self.built: - return - # Let's start with defining which files are to be downloaded. # Let's explore the remote folder. self.download_path = download_path @@ -112,6 +140,12 @@ class Build(object): # Checking if a valid path has been provided if self.url[arch] == '-': print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) + # Faking already built it so to skip other checks. + self.built[arch] = True + continue + + if self.built[arch]: + print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) continue contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") @@ -137,22 +171,15 @@ class Build(object): def build(self): """Building all the versions.""" - if self.built: - return - # We have 4 builds to do: # * standard languages, no help # * standard languages + offline help # * all languages, no help # * all languages + offline help - if self.portable and not 'portable' in self.storage_path: - self.storage_path += "/portable" - for arch in self.arch: - # Checking if a valid path has been provided - if self.url[arch] == '-': - # User has been warned in download step. + if self.built[arch]: + # Already built for arch or path not available. User has already been warned. continue # Preparation tasks @@ -261,9 +288,13 @@ class Build(object): def checksums(self): """Create checksums of the built versions.""" - if self.built: + if all(self.built.values()): + # All checksums are already created. return + # On the contrary, checksums will be in any case overwritten if + # existent, but generated only for built packages anyways + os.chdir(self.appnamedir) for appimage in glob.glob('*.AppImage*'): # See if a checksum already exist @@ -271,13 +302,16 @@ class Build(object): subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) - def move(self): + def publish(self): """Moves built versions to definitive storage.""" - if self.built: + if all(self.built.values()): + # All files are already present in the full_path return os.chdir(self.appnamedir) - subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.storage_path, shell=True) + # Forcing creation of subfolders, in case there is a new build + os.makedirs(self.full_path, exist_ok = True) + subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) def __del__(self): """Destructor""" diff --git a/scripts/loaih-build b/scripts/loaih-build index 111e64f..69158c9 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -37,7 +37,7 @@ def build(arch, language, offline, portable, updatable, download, storage, check obj.download(download) obj.build() obj.checksums() - obj.move() + obj.publish() del obj if __name__ == '__main__': diff --git a/setup.py b/setup.py index 0f9a07d..032157f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup,find_packages setup( name="loaih", - version="1.0.0", + version="1.0.1", description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", author="Emiliano Vavassori", author_email="syntaxerrormmm@libreoffice.org", From 8c630cbc560d94a4b11278d3029de361e6d4ee82 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 00:12:15 +0200 Subject: [PATCH 077/176] Aggiustato piccolo typo. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 55566b9..1fe8873 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -127,7 +127,7 @@ class Build(object): fullpath_arr = self.storage_path.split('/') # Joining relative path only if it is not null if len(self.relative_path) > 0: - fullpath_arr.expand(self.relative_path) + fullpath_arr.extend(self.relative_path) self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) def download(self, download_path): From cc31d82eb28e1cf637e83d7b3f01946727cfdfb3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 00:25:42 +0200 Subject: [PATCH 078/176] =?UTF-8?q?Trovato=20e=20fixato=20bug=20per=20cui?= =?UTF-8?q?=20basic=20sono=20pi=C3=B9=20grandi=20di=20standard.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loaih/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 1fe8873..c25c045 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -205,14 +205,16 @@ class Build(object): # Let's process standard languages and append results to the # buildtarball if self.language == 'basic': - buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'helppack_en-GB' in x ]) - if self.language == 'standard': + buildtarballs.extend([ x for x in self.tarballs[arch] if 'pack_en-GB' in x ]) + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) + elif self.language == 'standard': for lang in Build.LANGSTD: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('helppack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) else: # In any other cases, we build with all languages if not self.offline_help: @@ -284,7 +286,7 @@ class Build(object): # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) + subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) def checksums(self): """Create checksums of the built versions.""" From f2c64e195fc68892e0b99708bd6ce6544c5677c8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 00:42:31 +0200 Subject: [PATCH 079/176] Correzione file ver. Spero sistemato problema di lingua. --- loaih/__init__.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index c25c045..6964d1a 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -215,14 +215,20 @@ class Build(object): buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) else: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - else: - # In any other cases, we build with all languages - if not self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) - else: + elif self.language == 'full': + if self.offline_help: # We need also all help. Let's replace buildtarball with the # whole bunch buildtarballs = self.tarballs[arch] + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) + else: + # Looping for each language in self.language + for lang in self.language.split(","): + if self.offline_help: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) # Unpacking the tarballs for archive in buildtarballs: @@ -268,7 +274,7 @@ class Build(object): if generalize: subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) # Build version file management - with open(self.appimagefilename[arch] + '.ver', 'w') as v: + with open(self.genappimagefilename[arch] + '.ver', 'w') as v: v.write(self.version) else: subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) @@ -276,7 +282,7 @@ class Build(object): else: if generalize: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.genappversion, appname = self.appname), shell=True) - with open(self.appimagefilename[arch] + '.ver', 'w') as v: + with open(self.genappimagefilename[arch] + '.ver', 'w') as v: v.write(self.version) else: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) From eade74c29a5d04ee15b8afa81ba9ab2c6926471f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 01:06:38 +0200 Subject: [PATCH 080/176] Script per gli update. Aggiunte directory all'ignore. Non creato checksum per .ver. --- .gitignore | 3 +++ check_updates.sh | 23 +++++++++++++++++++++++ loaih/__init__.py | 9 +++++++++ 3 files changed, 35 insertions(+) create mode 100755 check_updates.sh diff --git a/.gitignore b/.gitignore index 5ceb386..ff7d489 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ venv +build +dist +loaih.egg-info diff --git a/check_updates.sh b/check_updates.sh new file mode 100755 index 0000000..5820256 --- /dev/null +++ b/check_updates.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +pushd $(dirname $0) +# Checking if pull is successfull +git fetch origin +LC_ALL=C git status | grep -q "Your branch is up to date"; retval=$? + +if [[ ${retval} -ne 0 ]]; then + # Let us update the pip installation + git pull + # for the sake of consistency, let's make the check_updates.sh script + # executable + chmod +x check_updates.sh + pip3 uninstall -y loaih + # build the actual toolkit + python3 setup.py bdist_wheel + pip3 install dist/*.whl; rv=$? + + if [[ ${rv} -eq 0 ]]; then + # cleanup + rm -rf dist +fi +popd diff --git a/loaih/__init__.py b/loaih/__init__.py index 6964d1a..30161fa 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -199,6 +199,11 @@ class Build(object): self.__unpackbuild__(arch) def __unpackbuild__(self, arch, generalize = False): + if generalize and self.portable: + # Doesn't particularly make sense to build a generic portable + # version. Just skipping the specific generic build + return + # We start by filtering out tarballs from the list buildtarballs = [ self.tarballs[arch][0] ] @@ -305,6 +310,10 @@ class Build(object): os.chdir(self.appnamedir) for appimage in glob.glob('*.AppImage*'): + if appimage.endswith('.ver'): + # Skipping checksums for .ver files. + continue + # See if a checksum already exist if not os.path.exists(appimage + '.md5'): subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) From 6895ec357b9878dfa42501e24af79648460ece9e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 01:08:39 +0200 Subject: [PATCH 081/176] Corretto script di check_updates.sh. --- check_updates.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/check_updates.sh b/check_updates.sh index 5820256..fd557b5 100755 --- a/check_updates.sh +++ b/check_updates.sh @@ -18,6 +18,7 @@ if [[ ${retval} -ne 0 ]]; then if [[ ${rv} -eq 0 ]]; then # cleanup - rm -rf dist + rm -rf dist build loaih.egg-info + fi fi popd From b21d14324a07b32176948a33d157a3a34831caf1 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 01:10:36 +0200 Subject: [PATCH 082/176] Trovato problema per build standard non corrette con le lingue. Corretto script di check_updates. --- loaih/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 30161fa..8776d02 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -217,9 +217,9 @@ class Build(object): elif self.language == 'standard': for lang in Build.LANGSTD: if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack_' + lang) in x ]) else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack_' + lang) in x ]) elif self.language == 'full': if self.offline_help: # We need also all help. Let's replace buildtarball with the From 42be51b82e9e24ad8ab9150f8b66d43b3ea1bacd Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 01:36:27 +0200 Subject: [PATCH 083/176] Implementato buildfile yml. --- fresh.yml | 65 +++++++++++++++++++++++++++++++++++++++++++++ scripts/loaih-build | 56 +++++++++++++++++++++++++++++--------- 2 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 fresh.yml diff --git a/fresh.yml b/fresh.yml new file mode 100644 index 0000000..1f8735a --- /dev/null +++ b/fresh.yml @@ -0,0 +1,65 @@ +--- +data: + storage: /srv/http/appimage.sys42.eu + download: /var/tmp/downloads + +builds: + - query: fresh + language: basic + offline_help: no + portable: no + + - query: fresh + language: basic + offline_help: yes + portable: no + + - query: fresh + language: basic + offline_help: no + portable: yes + + - query: fresh + language: basic + offline_help: yes + portable: yes + + - query: fresh + language: standard + offline_help: no + portable: no + + - query: fresh + language: standard + offline_help: yes + portable: no + + - query: fresh + language: standard + offline_help: no + portable: yes + + - query: fresh + language: standard + offline_help: yes + portable: yes + + - query: fresh + language: full + offline_help: no + portable: no + + - query: fresh + language: full + offline_help: yes + portable: no + + - query: fresh + language: full + offline_help: no + portable: yes + + - query: fresh + language: full + offline_help: yes + portable: yes diff --git a/scripts/loaih-build b/scripts/loaih-build index 69158c9..f539753 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -2,6 +2,7 @@ # encoding: utf-8 import click +import yaml import loaih @click.command() @@ -22,23 +23,52 @@ def build(arch, language, offline, portable, updatable, download, storage, check else: arches = [ arch.lower().decode('utf-8') ] - obj = loaih.Build(query, arches) + if query.endswith('.yml') or query.endswith('.yaml'): + # This is a buildfile. So we have to load the file and pass the build options ourselves. + config = {} + with open(query, 'r') as file: + config = yaml.safe_load(file) - obj.language = language - obj.offline_help = offline - obj.portable = portable - obj.updatable = updatable + # With the config file, we ignore all the command line options and set + # generic default. + for build in config['builds']: + # Loop a run for each build. + obj = loaih.Build(build['query'], [u'x86', u'x86_64']) + + obj.language = build['language'] + obj.offline_help = build['offline_help'] + obj.portable = build['portable'] + obj.updatable = True + + if check: + obj.check(config['data']['storage']) + else: + obj.storage_path = config['data']['storage'] + + obj.download(config['data']['download']) + obj.build() + obj.checksums() + obj.publish() + del obj - if check: - obj.check(storage) else: - obj.storage_path = storage + obj = loaih.Build(query, arches) - obj.download(download) - obj.build() - obj.checksums() - obj.publish() - del obj + obj.language = language + obj.offline_help = offline + obj.portable = portable + obj.updatable = updatable + + if check: + obj.check(storage) + else: + obj.storage_path = storage + + obj.download(download) + obj.build() + obj.checksums() + obj.publish() + del obj if __name__ == '__main__': build() From 96907ad24b5ba3e251c0aed0c274b8c8d1a0a87e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 01:47:00 +0200 Subject: [PATCH 084/176] Piccoli fix cosmetici allo script di build. --- scripts/loaih-build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/loaih-build b/scripts/loaih-build index f539753..5ff89a2 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -17,6 +17,7 @@ import loaih @click.argument('query') def build(arch, language, offline, portable, updatable, download, storage, check, query): # Parsing options + arches = [] if arch.lower() == 'all': # We need to build it twice. arches = [ u'x86', u'x86_64' ] @@ -33,7 +34,7 @@ def build(arch, language, offline, portable, updatable, download, storage, check # generic default. for build in config['builds']: # Loop a run for each build. - obj = loaih.Build(build['query'], [u'x86', u'x86_64']) + obj = loaih.Build(build['query'], arches) obj.language = build['language'] obj.offline_help = build['offline_help'] From d5f118f30494660b8653a8e07969bc44cd44116d Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 01:51:13 +0200 Subject: [PATCH 085/176] Adding script for building still. --- still.yml | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 still.yml diff --git a/still.yml b/still.yml new file mode 100644 index 0000000..3d86642 --- /dev/null +++ b/still.yml @@ -0,0 +1,65 @@ +--- +data: + storage: /srv/http/appimage.sys42.eu + download: /var/tmp/downloads + +builds: + - query: still + language: basic + offline_help: no + portable: no + + - query: still + language: basic + offline_help: yes + portable: no + + - query: still + language: basic + offline_help: no + portable: yes + + - query: still + language: basic + offline_help: yes + portable: yes + + - query: still + language: standard + offline_help: no + portable: no + + - query: still + language: standard + offline_help: yes + portable: no + + - query: still + language: standard + offline_help: no + portable: yes + + - query: still + language: standard + offline_help: yes + portable: yes + + - query: still + language: full + offline_help: no + portable: no + + - query: still + language: full + offline_help: yes + portable: no + + - query: still + language: full + offline_help: no + portable: yes + + - query: still + language: full + offline_help: yes + portable: yes From 11570940f7a9f6db40c5ebfda4a1b553123e1add Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 30 Mar 2022 02:06:24 +0200 Subject: [PATCH 086/176] Evito di chiamare una funzione con un controllo if. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 8776d02..89f05d1 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -194,7 +194,7 @@ class Build(object): os.chmod('appimagetool', 0o755) # Build the requested version. - if self.queried_name: + if self.queried_name and not self.portable: self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) From 651022622780ff0d6a235b9521fdc0d1d043522f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 00:24:21 +0200 Subject: [PATCH 087/176] Primo push con nuova versione del build. --- scripts/loaih-build | 15 ++++++++------- setup.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/loaih-build b/scripts/loaih-build index 5ff89a2..033d9dc 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -7,15 +7,16 @@ import loaih @click.command() @click.option('-a', '--arch', 'arch', type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), default='all', help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") +@click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") +@click.option('-d', '--download-path', 'download_path', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") @click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") +@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") -@click.option('-d', '--download-path', 'download', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") -@click.option('-s', '--storage-path', 'storage', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") -@click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") @click.argument('query') -def build(arch, language, offline, portable, updatable, download, storage, check, query): +def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): # Parsing options arches = [] if arch.lower() == 'all': @@ -61,11 +62,11 @@ def build(arch, language, offline, portable, updatable, download, storage, check obj.updatable = updatable if check: - obj.check(storage) + obj.check(repo_path) else: - obj.storage_path = storage + obj.storage_path = repo_path - obj.download(download) + obj.download(download_path) obj.build() obj.checksums() obj.publish() diff --git a/setup.py b/setup.py index 032157f..4ee9309 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup,find_packages setup( name="loaih", - version="1.0.1", + version="1.1.0", description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", author="Emiliano Vavassori", author_email="syntaxerrormmm@libreoffice.org", From a68882888a2a16cec6c6cc3cec0be1c96c940f90 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 01:00:13 +0200 Subject: [PATCH 088/176] Correzione script di nuovo. --- scripts/loaih-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/loaih-build b/scripts/loaih-build index 033d9dc..80b6295 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -21,7 +21,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path arches = [] if arch.lower() == 'all': # We need to build it twice. - arches = [ u'x86', u'x86_64' ] + arches = [ 'x86', 'x86_64' ] else: arches = [ arch.lower().decode('utf-8') ] From 5ce7f5cc071694929eb23d30da8d5844fe7d9853 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 01:03:32 +0200 Subject: [PATCH 089/176] Rimozione forzatura stringa utf-8 da libreria. --- loaih/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 89f05d1..0d2c402 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -35,7 +35,7 @@ class Build(object): self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' self.version = v.version self.url = v.basedirurl - self.built = { u'x86': False, u'x86_64': False } + self.built = { 'x86': False, 'x86_64': False } # Preparing the default for the relative path on the storage for # different versions. @@ -73,12 +73,12 @@ class Build(object): # If the build was called by queried name, build from latest release available but build with the most generic name myver = str.join('.', self.version.split('.')[0:2]) self.genappversion = myver + self.languagepart + self.helppart - self.genappimagefilename[u'x86'] = self.appname + '-' + self.genappversion + '-x86.AppImage' - self.genappimagefilename[u'x86_64'] = self.appname + '-' + self.genappversion + '-x86_64.AppImage' + self.genappimagefilename['x86'] = self.appname + '-' + self.genappversion + '-x86.AppImage' + self.genappimagefilename['x86_64'] = self.appname + '-' + self.genappversion + '-x86_64.AppImage' self.appversion = self.version + self.languagepart + self.helppart - self.appimagefilename[u'x86'] = self.appname + '-' + self.appversion + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + self.appversion + '-x86_64.AppImage' + self.appimagefilename['x86'] = self.appname + '-' + self.appversion + '-x86.AppImage' + self.appimagefilename['x86_64'] = self.appname + '-' + self.appversion + '-x86_64.AppImage' for arch in self.arch: # For generalized builds, we need to check if there are .ver file From c88e10b2acf2383e00d5a6e38aafc4c7930e270b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 01:19:51 +0200 Subject: [PATCH 090/176] Refactoring di determinato codice, vediamo se potrebbe funzionare. --- loaih/__init__.py | 11 ++++----- scripts/loaih-build | 60 ++++++++++++--------------------------------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 0d2c402..ae3ecf8 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -8,6 +8,7 @@ import tempfile, os, sys, glob, subprocess, shutil, re class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGBASIC = [ 'en-GB' ] + ARCHSTD = [ u'x86', u'x86_64' ] def __init__(self, query, arch): """Build all versions that can be found in the indicated repo.""" @@ -35,7 +36,7 @@ class Build(object): self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' self.version = v.version self.url = v.basedirurl - self.built = { 'x86': False, 'x86_64': False } + self.built = { u'x86': False, u'x86_64': False } # Preparing the default for the relative path on the storage for # different versions. @@ -73,12 +74,10 @@ class Build(object): # If the build was called by queried name, build from latest release available but build with the most generic name myver = str.join('.', self.version.split('.')[0:2]) self.genappversion = myver + self.languagepart + self.helppart - self.genappimagefilename['x86'] = self.appname + '-' + self.genappversion + '-x86.AppImage' - self.genappimagefilename['x86_64'] = self.appname + '-' + self.genappversion + '-x86_64.AppImage' - self.appversion = self.version + self.languagepart + self.helppart - self.appimagefilename['x86'] = self.appname + '-' + self.appversion + '-x86.AppImage' - self.appimagefilename['x86_64'] = self.appname + '-' + self.appversion + '-x86_64.AppImage' + for arch in ARCHSTD: + self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' + self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' for arch in self.arch: # For generalized builds, we need to check if there are .ver file diff --git a/scripts/loaih-build b/scripts/loaih-build index 80b6295..3b704b4 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -2,7 +2,6 @@ # encoding: utf-8 import click -import yaml import loaih @click.command() @@ -21,56 +20,27 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path arches = [] if arch.lower() == 'all': # We need to build it twice. - arches = [ 'x86', 'x86_64' ] + arches = [ u'x86', u'x86_64' ] else: - arches = [ arch.lower().decode('utf-8') ] + arches = [ arch.lower() ] - if query.endswith('.yml') or query.endswith('.yaml'): - # This is a buildfile. So we have to load the file and pass the build options ourselves. - config = {} - with open(query, 'r') as file: - config = yaml.safe_load(file) + obj = loaih.Build(query, arches) - # With the config file, we ignore all the command line options and set - # generic default. - for build in config['builds']: - # Loop a run for each build. - obj = loaih.Build(build['query'], arches) - - obj.language = build['language'] - obj.offline_help = build['offline_help'] - obj.portable = build['portable'] - obj.updatable = True - - if check: - obj.check(config['data']['storage']) - else: - obj.storage_path = config['data']['storage'] - - obj.download(config['data']['download']) - obj.build() - obj.checksums() - obj.publish() - del obj + obj.language = language + obj.offline_help = offline + obj.portable = portable + obj.updatable = updatable + if check: + obj.check(repo_path) else: - obj = loaih.Build(query, arches) + obj.storage_path = repo_path - obj.language = language - obj.offline_help = offline - obj.portable = portable - obj.updatable = updatable - - if check: - obj.check(repo_path) - else: - obj.storage_path = repo_path - - obj.download(download_path) - obj.build() - obj.checksums() - obj.publish() - del obj + obj.download(download_path) + obj.build() + obj.checksums() + obj.publish() + del obj if __name__ == '__main__': build() From 2c60de03cb91a7aa929ef502a7d739c809fd6bf3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 01:28:06 +0200 Subject: [PATCH 091/176] Ancora correzioni allo script di dev. --- loaih/__init__.py | 2 +- scripts/loaih-build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index ae3ecf8..12b9154 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -193,7 +193,7 @@ class Build(object): os.chmod('appimagetool', 0o755) # Build the requested version. - if self.queried_name and not self.portable: + if self.queried_name: self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) diff --git a/scripts/loaih-build b/scripts/loaih-build index 3b704b4..8139578 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -12,7 +12,7 @@ import loaih @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") @click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") -@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Whether to sign the build. Default: no-sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): From 544d805b5018f3c12ef4bf551112ac636e43eae7 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 01:50:15 +0200 Subject: [PATCH 092/176] =?UTF-8?q?Commit=20con=20funzionalit=C3=A0=20di?= =?UTF-8?q?=20firma=20complete.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loaih/__init__.py | 59 +++++++++++++++++++++++++--------------- scripts/loaih-build | 66 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 89 insertions(+), 36 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 12b9154..1c64989 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -20,11 +20,14 @@ class Build(object): self.offline_help = False self.portable = False self.updatable = True + self.sign = False self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' + + # Specific build version self.appversion = '' - self.appimagefilename = {} self.genappversion = '' + self.appimagefilename = {} self.genappimagefilename = {} # Getting versions and so on @@ -45,6 +48,26 @@ class Build(object): self.relative_path = [] self.full_path = '' + def version_strings(self): + """Build version strings based on the information received.""" + # Building expected AppImageName + self.languagepart = "." + if ',' in self.language: + self.languagepart += self.language.replace(',', '-') + else: + self.languagepart += self.language + + self.helppart = '.help' if self.offline_help else '' + + # If the build was called by queried name, build from latest release available but build with the most generic name + self.appversion = self.version + self.languagepart + self.helppart + myver = str.join('.', self.version.split('.')[0:2]) + self.genappversion = myver + self.languagepart + self.helppart + for arch in ARCHSTD: + self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' + self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' + + def check(self, storage_path): """Checking if the requested AppImage has been already built.""" # Mandate to the private function to calculate the full_path available @@ -62,22 +85,8 @@ class Build(object): # to build an updatable version. self.updatable = False - # Building expected AppImageName - self.languagepart = "." - if ',' in self.language: - self.languagepart += self.language.replace(',', '-') - else: - self.languagepart += self.language - - self.helppart = '.help' if self.offline_help else '' - - # If the build was called by queried name, build from latest release available but build with the most generic name - myver = str.join('.', self.version.split('.')[0:2]) - self.genappversion = myver + self.languagepart + self.helppart - self.appversion = self.version + self.languagepart + self.helppart - for arch in ARCHSTD: - self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' - self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' + # Call the build for queries + self.version_strings() for arch in self.arch: # For generalized builds, we need to check if there are .ver file @@ -193,7 +202,8 @@ class Build(object): os.chmod('appimagetool', 0o755) # Build the requested version. - if self.queried_name: + if self.queried_name and not self.portable: + # If it is portable, do not generate a generalized version self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) @@ -274,22 +284,27 @@ class Build(object): # updatable version. # zsync name was generated already + # Dealing with extra options + buildopts = [] + if self.sign: + buildopts.append('--sign') + # If asked to do a generalized build: if generalize: - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) # Build version file management with open(self.genappimagefilename[arch] + '.ver', 'w') as v: v.write(self.version) else: - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) else: if generalize: - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.genappversion, appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) with open(self.genappimagefilename[arch] + '.ver', 'w') as v: v.write(self.version) else: - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) + subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) print("Built AppImage version {version}".format(version = self.appversion)) diff --git a/scripts/loaih-build b/scripts/loaih-build index 8139578..76de886 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -2,6 +2,7 @@ # encoding: utf-8 import click +import yaml import loaih @click.command() @@ -12,7 +13,7 @@ import loaih @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") @click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") -@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Whether to sign the build. Default: no-sign") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): @@ -24,23 +25,60 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path else: arches = [ arch.lower() ] - obj = loaih.Build(query, arches) + if query.endswith('.yml') or query.endswith('.yaml'): + # This is a buildfile. So we have to load the file and pass the build options ourselves. + config = {} + with open(query, 'r') as file: + config = yaml.safe_load(file) - obj.language = language - obj.offline_help = offline - obj.portable = portable - obj.updatable = updatable + # With the config file, we ignore all the command line options and set + # generic default. + for build in config['builds']: + # Loop a run for each build. + obj = loaih.Build(build['query'], arches) + + obj.language = build['language'] + obj.offline_help = build['offline_help'] + obj.portable = build['portable'] + obj.updatable = True + + if 'sign' in config['data'] and config['data']['sign']: + obj.sign = True + + if 'force' in config['data'] and config['data']['force']: + obj.storage_path = config['data']['repo'] + obj.version_strings() + else: + obj.check(config['data']['repo']) + + obj.download(config['data']['download']) + obj.build() + obj.checksums() + obj.publish() + del obj - if check: - obj.check(repo_path) else: - obj.storage_path = repo_path + obj = loaih.Build(query, arches) - obj.download(download_path) - obj.build() - obj.checksums() - obj.publish() - del obj + obj.language = language + obj.offline_help = offline + obj.portable = portable + obj.updatable = updatable + + if sign: + obj.sign = True + + if check: + obj.check(repo_path) + else: + obj.storage_path = repo_path + obj.version_strings() + + obj.download(download_path) + obj.build() + obj.checksums() + obj.publish() + del obj if __name__ == '__main__': build() From a8e522e12dced3926aba4d1a6f8f5b0d365a54cb Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 01:51:37 +0200 Subject: [PATCH 093/176] Correzione piccolo problema di richiamo costante. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 1c64989..3eb19c5 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -63,7 +63,7 @@ class Build(object): self.appversion = self.version + self.languagepart + self.helppart myver = str.join('.', self.version.split('.')[0:2]) self.genappversion = myver + self.languagepart + self.helppart - for arch in ARCHSTD: + for arch in Build.ARCHSTD: self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' From b8b80ebdea1d68f243148d1209f9b4ec707506ba Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 02:02:15 +0200 Subject: [PATCH 094/176] Cambiato lo standard dei file di build per allinearsi con l'opzione sign. --- fresh.yml | 4 +++- still.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fresh.yml b/fresh.yml index 1f8735a..146e19e 100644 --- a/fresh.yml +++ b/fresh.yml @@ -1,7 +1,9 @@ --- data: - storage: /srv/http/appimage.sys42.eu + repo: /srv/http/appimage.sys42.eu download: /var/tmp/downloads + force: no + sign: no builds: - query: fresh diff --git a/still.yml b/still.yml index 3d86642..bfe19b7 100644 --- a/still.yml +++ b/still.yml @@ -1,7 +1,9 @@ --- data: - storage: /srv/http/appimage.sys42.eu + repo: /srv/http/appimage.sys42.eu download: /var/tmp/downloads + force: no + sign: no builds: - query: still From 7ae5ac182a36eff4ab8bf7885a0b20f40ad52d67 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 02:12:46 +0200 Subject: [PATCH 095/176] Correzione logica di esecuzione e assegnazione attributi. --- loaih/__init__.py | 44 +++++++++++++++++++------------------------- scripts/loaih-build | 29 +++++++++++++++++------------ 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 3eb19c5..ea0728a 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -48,8 +48,23 @@ class Build(object): self.relative_path = [] self.full_path = '' - def version_strings(self): - """Build version strings based on the information received.""" + def calculate(self): + """Calculate exclusions and other variables.""" + # Incompatibilities - if portable and updatable are asked together, + # only portable will be built. + if self.portable and self.updatable: + print("Upgradable and portable options were required together. Building only portable.") + self.updatable = False + + if self.updatable and not self.queried_name: + # If the queried version was a numbered version, doesn't make sense + # to build an updatable version. + self.updatable = False + + # Mandate to the private function to calculate the full_path available + # for the storage and the checks. + self.__calculate_full_path__() + # Building expected AppImageName self.languagepart = "." if ',' in self.language: @@ -68,25 +83,8 @@ class Build(object): self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' - def check(self, storage_path): + def check(self): """Checking if the requested AppImage has been already built.""" - # Mandate to the private function to calculate the full_path available - # for the storage and the checks. - self.__calculate_full_path__() - - # Incompatibilities - if portable and updatable are asked together, - # only portable will be built. - if self.portable and self.updatable: - print("Upgradable and portable options were required together. Building only portable.") - self.updatable = False - - if self.updatable and not self.queried_name: - # If the queried version was a numbered version, doesn't make sense - # to build an updatable version. - self.updatable = False - - # Call the build for queries - self.version_strings() for arch in self.arch: # For generalized builds, we need to check if there are .ver file @@ -138,12 +136,8 @@ class Build(object): fullpath_arr.extend(self.relative_path) self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) - def download(self, download_path): + def download(self): """Downloads the contents of the URL as it was a folder.""" - # Let's start with defining which files are to be downloaded. - # Let's explore the remote folder. - self.download_path = download_path - for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': diff --git a/scripts/loaih-build b/scripts/loaih-build index 76de886..7a1ead4 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -37,21 +37,23 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path # Loop a run for each build. obj = loaih.Build(build['query'], arches) + # Configuration phase obj.language = build['language'] obj.offline_help = build['offline_help'] obj.portable = build['portable'] obj.updatable = True + obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' + obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' if 'sign' in config['data'] and config['data']['sign']: obj.sign = True - if 'force' in config['data'] and config['data']['force']: - obj.storage_path = config['data']['repo'] - obj.version_strings() - else: - obj.check(config['data']['repo']) + # Build phase + obj.calculate() + if not 'force' in config['data'] or not config['data']['force']: + obj.check() - obj.download(config['data']['download']) + obj.download() obj.build() obj.checksums() obj.publish() @@ -60,21 +62,24 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path else: obj = loaih.Build(query, arches) + # Configuration phase obj.language = language obj.offline_help = offline obj.portable = portable obj.updatable = updatable + obj.storage_path = repo_path + obj.download_path = download_path if sign: obj.sign = True - if check: - obj.check(repo_path) - else: - obj.storage_path = repo_path - obj.version_strings() + # Running phase + obj.calculate() - obj.download(download_path) + if check: + obj.check() + + obj.download() obj.build() obj.checksums() obj.publish() From 72c36521f670f77b2fa22e2cdc50251debc5a748 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 02:53:45 +0200 Subject: [PATCH 096/176] Nei buildfile firmo per default. --- fresh.yml | 2 +- still.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fresh.yml b/fresh.yml index 146e19e..8678b0f 100644 --- a/fresh.yml +++ b/fresh.yml @@ -3,7 +3,7 @@ data: repo: /srv/http/appimage.sys42.eu download: /var/tmp/downloads force: no - sign: no + sign: yes builds: - query: fresh diff --git a/still.yml b/still.yml index bfe19b7..2a97d14 100644 --- a/still.yml +++ b/still.yml @@ -3,7 +3,7 @@ data: repo: /srv/http/appimage.sys42.eu download: /var/tmp/downloads force: no - sign: no + sign: yes builds: - query: still From 2f8b4c444af2d2523c47584aa37a33a407bfd3d8 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 31 Mar 2022 09:38:00 +0200 Subject: [PATCH 097/176] Sistemazione logica per costruzione buildopts. --- loaih/__init__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index ea0728a..a754aff 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -48,6 +48,7 @@ class Build(object): self.relative_path = [] self.full_path = '' + def calculate(self): """Calculate exclusions and other variables.""" # Incompatibilities - if portable and updatable are asked together, @@ -118,6 +119,7 @@ class Build(object): if self.built[arch]: print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) + def __calculate_full_path__(self): """Calculate relative path of the build, based on internal other variables.""" if len(self.relative_path) == 0: @@ -136,6 +138,7 @@ class Build(object): fullpath_arr.extend(self.relative_path) self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) + def download(self): """Downloads the contents of the URL as it was a folder.""" for arch in self.arch: @@ -171,6 +174,7 @@ class Build(object): print("Got %s." % archive) + def build(self): """Building all the versions.""" # We have 4 builds to do: @@ -201,6 +205,7 @@ class Build(object): self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) + def __unpackbuild__(self, arch, generalize = False): if generalize and self.portable: # Doesn't particularly make sense to build a generic portable @@ -270,6 +275,11 @@ class Build(object): dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) + + # Dealing with extra options + buildopts = [] + if self.sign: + buildopts.append('--sign') # Building app if self.updatable: @@ -278,11 +288,6 @@ class Build(object): # updatable version. # zsync name was generated already - # Dealing with extra options - buildopts = [] - if self.sign: - buildopts.append('--sign') - # If asked to do a generalized build: if generalize: subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) @@ -307,6 +312,7 @@ class Build(object): os.remove(deb) subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) + def checksums(self): """Create checksums of the built versions.""" if all(self.built.values()): @@ -338,6 +344,7 @@ class Build(object): os.makedirs(self.full_path, exist_ok = True) subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) + def __del__(self): """Destructor""" # Cleaning up build directory From 6387ad1ac0cba92cd3037f14143bec388d59054c Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 7 Apr 2022 02:34:06 +0200 Subject: [PATCH 098/176] Aggiornamento storage. --- fresh.yml | 2 +- still.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fresh.yml b/fresh.yml index 8678b0f..e8ba85d 100644 --- a/fresh.yml +++ b/fresh.yml @@ -1,6 +1,6 @@ --- data: - repo: /srv/http/appimage.sys42.eu + repo: /mnt/appimage download: /var/tmp/downloads force: no sign: yes diff --git a/still.yml b/still.yml index 2a97d14..3f5a595 100644 --- a/still.yml +++ b/still.yml @@ -1,6 +1,6 @@ --- data: - repo: /srv/http/appimage.sys42.eu + repo: /mnt/appimage download: /var/tmp/downloads force: no sign: yes From 413b84a1c322e3675143e1ffc648c335c0877318 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 00:56:28 +0200 Subject: [PATCH 099/176] Fix tentativo per creazione di link corretti e zsync truccati. --- loaih/__init__.py | 276 +++++++++++++++++++++----------------------- scripts/loaih-build | 3 +- 2 files changed, 134 insertions(+), 145 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index a754aff..3c20115 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -3,7 +3,7 @@ import urllib.request import loaih.versions as versions from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re +import tempfile, os, sys, glob, subprocess, shutil, re, shlex class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] @@ -13,32 +13,34 @@ class Build(object): def __init__(self, query, arch): """Build all versions that can be found in the indicated repo.""" self.query = query - self.queried_name = False if '.' in self.query else True self.arch = arch - self.url = {} + + # Getting versions and so on + v = versions.BuildVersion(self.query) + self.version = v.version + self.short_version = str.join('.', self.version.split('.')[0:2]) + self.branch_version = None + if not '.' in self.query: + self.branch_version = self.query + self.url = v.basedirurl + + # Other default values self.language = 'basic' self.offline_help = False self.portable = False self.updatable = True - self.sign = False - self.storage_path = '/srv/http/appimage.sys42.eu' + self.sign = True + self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' # Specific build version self.appversion = '' - self.genappversion = '' self.appimagefilename = {} - self.genappimagefilename = {} - - # Getting versions and so on - v = versions.BuildVersion(self.query) + self.zsyncfilename = {} # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' - self.version = v.version - self.url = v.basedirurl self.built = { u'x86': False, u'x86_64': False } # Preparing the default for the relative path on the storage for @@ -51,73 +53,32 @@ class Build(object): def calculate(self): """Calculate exclusions and other variables.""" - # Incompatibilities - if portable and updatable are asked together, - # only portable will be built. - if self.portable and self.updatable: - print("Upgradable and portable options were required together. Building only portable.") - self.updatable = False + # AppName + self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' - if self.updatable and not self.queried_name: - # If the queried version was a numbered version, doesn't make sense - # to build an updatable version. - self.updatable = False - - # Mandate to the private function to calculate the full_path available - # for the storage and the checks. - self.__calculate_full_path__() - - # Building expected AppImageName + # Calculating languagepart self.languagepart = "." if ',' in self.language: self.languagepart += self.language.replace(',', '-') else: self.languagepart += self.language + # Calculating help part self.helppart = '.help' if self.offline_help else '' - # If the build was called by queried name, build from latest release available but build with the most generic name - self.appversion = self.version + self.languagepart + self.helppart - myver = str.join('.', self.version.split('.')[0:2]) - self.genappversion = myver + self.languagepart + self.helppart + # Building the required names for arch in Build.ARCHSTD: - self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' - self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' + self.appimagefilename[arch] = __genappimagefilename__(self.version, arch) + self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' + + # Mandate to the private function to calculate the full_path available + # for the storage and the checks. + self.__calculate_full_path__() - def check(self): - """Checking if the requested AppImage has been already built.""" - - for arch in self.arch: - # For generalized builds, we need to check if there are .ver file - # and it contains the specific version found. - print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) - res = subprocess.run("find {path} -name {appimage}'".format( - path = self.full_path, - appimage = self.genappimagefilename[arch] + '.ver' - ), shell=True, capture_output=True, env={ "LC_ALL": "C" }) - - if "No such file or directory" in res.stderr.decode('utf-8'): - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue - - if res.stdout: - # All good, the command was executed fine. - for file in res.stdout.decode('utf-8').strip('\n').split('\n'): - if self.version in open(file, 'r').read(): - self.built[arch] = True - - print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) - res = subprocess.run("find {path} -name '{appimage}'".format( - path = self.full_path, - appimage = self.appimagefilename[arch] - ), shell=True, capture_output=True) - if res.stdout: - if len(res.stdout.decode('utf-8').strip('\n')) > 1: - self.built[arch] = True - - if self.built[arch]: - print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) + def __gen_appimagefilename__(self, version, arch): + """Generalize the construction of the name of the app.""" + return self.appname + f'-{version}' + self.languagepart + self.helppart + f'-{arch}.AppImage' def __calculate_full_path__(self): @@ -139,49 +100,70 @@ class Build(object): self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) + def check(self): + """Checking if the requested AppImage has been already built.""" + if not len(self.appimagefilename) == 2: + self.calculate() + + for arch in self.arch: + print(f"Searching for {self.appimagefilename[arch]}") + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}'"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + + if "No such file or directory" in res.stderr: + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout: + # All good, the command was executed fine. + for file in res.stdout.strip('\n').split('\n'): + if self.version in open(file, 'r').read(): + print(f"Build for {self.version} found.") + self.built[arch] = True + + if self.built[arch]: + print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + + def download(self): """Downloads the contents of the URL as it was a folder.""" + print(f"Started downloads for {self.version}. Please wait.") for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': - print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) + print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.") # Faking already built it so to skip other checks. self.built[arch] = True continue if self.built[arch]: - print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) + print(f"A build for {arch} was already found. Skipping specific packages.") continue + # Identifying downloads contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] tarballs = self.tarballs[arch] maintarball = tarballs[0] + # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) for archive in tarballs: # If the archive is already there, do not do anything. - if os.path.exists(os.path.join(self.download_path, archive)): - print("Archive %s is already there! Sweet" % archive) + if os.path.exists(archive): continue # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) except: - print("Failed to download {archive}.".format(archive = archive)) - - print("Got %s." % archive) + print(f"Failed to download {archive}.") + print(f"Finished downloads for {self.version}.") def build(self): """Building all the versions.""" - # We have 4 builds to do: - # * standard languages, no help - # * standard languages + offline help - # * all languages, no help - # * all languages + offline help for arch in self.arch: if self.built[arch]: @@ -190,28 +172,18 @@ class Build(object): # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) - self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') - os.makedirs(self.appimagedir, exist_ok = True) # And then cd to the appname folder. os.chdir(self.appnamedir) # Download appimagetool from github - appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) + appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage" urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') os.chmod('appimagetool', 0o755) # Build the requested version. - if self.queried_name and not self.portable: - # If it is portable, do not generate a generalized version - self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) - def __unpackbuild__(self, arch, generalize = False): - if generalize and self.portable: - # Doesn't particularly make sense to build a generic portable - # version. Just skipping the specific generic build - return - + def __unpackbuild__(self, arch): # We start by filtering out tarballs from the list buildtarballs = [ self.tarballs[arch][0] ] @@ -243,35 +215,38 @@ class Build(object): else: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - # Unpacking the tarballs - for archive in buildtarballs: - subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) - os.chdir(self.appnamedir) + # Unpacking the tarballs + for archive in buildtarballs: + subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + + # create appimagedir + self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) + # At this point, let's decompress the deb packages - subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) if self.portable: - shortversion = str.join('.', self.version.split('.')[:3]) - subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) # Changing desktop file - subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname), cwd=self.appimagedir) - subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) # Find the name of the binary called in the desktop file. - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') + binary_exec = subprocess.run(shlex.split("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') + binaryname = binary_exec.stdout.strip("\n") bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) + subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) # Download AppRun from github - apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) + apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) @@ -280,57 +255,36 @@ class Build(object): buildopts = [] if self.sign: buildopts.append('--sign') - - # Building app + + # adding zsync build if updatable if self.updatable: - # Updatable make sense only for generic images for fresh, still, - # daily. If a request was for a specific version, I'd not build an - # updatable version. - # zsync name was generated already + buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'") - # If asked to do a generalized build: - if generalize: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) - # Build version file management - with open(self.genappimagefilename[arch] + '.ver', 'w') as v: - v.write(self.version) - else: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) - - else: - if generalize: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) - with open(self.genappimagefilename[arch] + '.ver', 'w') as v: - v.write(self.version) - else: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) - - print("Built AppImage version {version}".format(version = self.appversion)) + buildopts_str = str.join(' ', buildopts) + # Build the number-specific build + subprocess.run(shlex.split(f"VERSION={self.appversion} ./appimagetool {buildopts_str} -v ./{self.appname}.AppDir/")) + + print(f"Built AppImage version {self.appversion}") # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) + subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) def checksums(self): """Create checksums of the built versions.""" + # Skip checksum if initally the build was already found in the storage directory if all(self.built.values()): - # All checksums are already created. - return - - # On the contrary, checksums will be in any case overwritten if - # existent, but generated only for built packages anyways + return os.chdir(self.appnamedir) - for appimage in glob.glob('*.AppImage*'): - if appimage.endswith('.ver'): - # Skipping checksums for .ver files. - continue - - # See if a checksum already exist - if not os.path.exists(appimage + '.md5'): - subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) + for arch in self.arch: + for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: + # For any built arch, find out if a file exist. + if len(glob.glob(self.appimagefilename[arch] + '.md5')) == 0: + # Build checksum + subprocess.run(shlex.split(f"md5sum {item} > {item}.md5")) def publish(self): @@ -342,7 +296,41 @@ class Build(object): os.chdir(self.appnamedir) # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) - subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) + for file in glob.glob("*.AppImage*"): + subprocess.run(shlex.split(f"cp -f {file} {self.fullpath}")) + + + def generalize_and_link(self): + """Creates the needed generalized files if needed.""" + # If called with a pointed version, no generalize and link necessary. + if not self.branch_version: + return + + # Creating versions for short version and query text + versions = [ self.short_version, self.branch_version ] + for arch in Build.ARCHSTD: + # if the appimage for the reported arch is not found, skip to next + # arch + if not os.path.exists(self.appimagefilename[arch]): + continue + + # Doing it both for short_name and for branchname + for version in versions: + appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + zsyncfilename[arch] = appimagefilename[arch] + '.zsync' + + os.chdir(self.full_path) + # Create the symlink + os.symlink(self.appimagefilename[arch], appimagefilename[arch]) + # Create the checksum for the AppImage + subprocess.run(shlex.split("md5sum {item} > {item}.md5".format(item=appimagefilename[arch]))) + # Do not continue if no zsync file is provided. + if not self.updatable: + continue + + os.copy(self.zsyncfilename[arch], zsyncfilename[arch]) + # Editing the zsyncfile + subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) def __del__(self): diff --git a/scripts/loaih-build b/scripts/loaih-build index 7a1ead4..08c3892 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -12,7 +12,7 @@ import loaih @click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") -@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") +@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") @click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') @@ -57,6 +57,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.build() obj.checksums() obj.publish() + obj.generalize_and_link() del obj else: From c5fac68a0106f8f273642ae401baad7da4ecf1b5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:01:21 +0200 Subject: [PATCH 100/176] Correzione chiamata funzione privata per generazione nome file appimage. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 3c20115..f498f9e 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -68,7 +68,7 @@ class Build(object): # Building the required names for arch in Build.ARCHSTD: - self.appimagefilename[arch] = __genappimagefilename__(self.version, arch) + self.appimagefilename[arch] = __gen_appimagefilename__(self.version, arch) self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' # Mandate to the private function to calculate the full_path available From 648883d514f565b28bf8744eda5ca457886b4310 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:02:39 +0200 Subject: [PATCH 101/176] Ancora correzione sul richiamo funzione privata. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index f498f9e..7213a08 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -68,7 +68,7 @@ class Build(object): # Building the required names for arch in Build.ARCHSTD: - self.appimagefilename[arch] = __gen_appimagefilename__(self.version, arch) + self.appimagefilename[arch] = self.__gen_appimagefilename__(self.version, arch) self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' # Mandate to the private function to calculate the full_path available From dc620f4b09e384433cf19e5fdfda5ec5efacc349 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:05:14 +0200 Subject: [PATCH 102/176] Mancata creazione directory di build. --- loaih/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index 7213a08..65ecdef 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -172,6 +172,7 @@ class Build(object): # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) + os.makedirs(self.appnamedir, exist_ok=True) # And then cd to the appname folder. os.chdir(self.appnamedir) # Download appimagetool from github From ff51cf8e51d60bfe5c3b00457b4eccfac7bf822a Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:28:18 +0200 Subject: [PATCH 103/176] Cambio logica per trovare il binaryname. --- loaih/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 65ecdef..e3fe8e7 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -239,8 +239,16 @@ class Build(object): subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) # Find the name of the binary called in the desktop file. - binary_exec = subprocess.run(shlex.split("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') - binaryname = binary_exec.stdout.strip("\n") + binaryname = '' + with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: + a = d.readlines() + for line in a: + if re.match(r'^Exec', line): + binaryname = line.split('=')[-1].split(' ')[0] + # Esci al primo match + break + #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') + #binaryname = binary_exec.stdout.strip("\n") bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) From 94be1e6ac9ee339612ad5078541df152fae372a6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:32:10 +0200 Subject: [PATCH 104/176] Chiamata build con variabile d'ambiente. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index e3fe8e7..249a32d 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -271,7 +271,7 @@ class Build(object): buildopts_str = str.join(' ', buildopts) # Build the number-specific build - subprocess.run(shlex.split(f"VERSION={self.appversion} ./appimagetool {buildopts_str} -v ./{self.appname}.AppDir/")) + subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ 'VERSION': self.appversion }) print(f"Built AppImage version {self.appversion}") From b97939ede698a21e4bcdee49cb877b212c08285a Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:36:47 +0200 Subject: [PATCH 105/176] Ancora sostituzione stringhe. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 249a32d..caca32d 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -78,7 +78,7 @@ class Build(object): def __gen_appimagefilename__(self, version, arch): """Generalize the construction of the name of the app.""" - return self.appname + f'-{version}' + self.languagepart + self.helppart + f'-{arch}.AppImage' + return self.appname + f"-{version}" + self.languagepart + self.helppart + f'-{arch}.AppImage' def __calculate_full_path__(self): From bf1d9e044694a75145e54f3b8faf037dacb7dc3e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:41:17 +0200 Subject: [PATCH 106/176] Ancora su versione lanciata in ambiente. --- loaih/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index caca32d..63dc18a 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -86,7 +86,7 @@ class Build(object): if len(self.relative_path) == 0: if self.query == 'daily': self.relative_path.append('daily') - elif self.query == 'prerelease': + elif self.query == 'primageerelease': self.relative_path.append('prerelease') # Not the same check, an additional one @@ -271,7 +271,7 @@ class Build(object): buildopts_str = str.join(' ', buildopts) # Build the number-specific build - subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ 'VERSION': self.appversion }) + subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) print(f"Built AppImage version {self.appversion}") @@ -291,7 +291,7 @@ class Build(object): for arch in self.arch: for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: # For any built arch, find out if a file exist. - if len(glob.glob(self.appimagefilename[arch] + '.md5')) == 0: + if not os.path.exists(f"{item}.md5"): # Build checksum subprocess.run(shlex.split(f"md5sum {item} > {item}.md5")) @@ -306,7 +306,7 @@ class Build(object): # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.fullpath}")) + subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) def generalize_and_link(self): From 076d5389d6284ad238b950aea98221200d122843 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:45:42 +0200 Subject: [PATCH 107/176] Aggiunta variabile di debug. --- loaih/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 63dc18a..62b4c32 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -18,6 +18,7 @@ class Build(object): # Getting versions and so on v = versions.BuildVersion(self.query) self.version = v.version + print(f"Debug {self.version}") self.short_version = str.join('.', self.version.split('.')[0:2]) self.branch_version = None if not '.' in self.query: @@ -78,7 +79,7 @@ class Build(object): def __gen_appimagefilename__(self, version, arch): """Generalize the construction of the name of the app.""" - return self.appname + f"-{version}" + self.languagepart + self.helppart + f'-{arch}.AppImage' + return self.appname + "-" + version + self.languagepart + self.helppart + f'-{arch}.AppImage' def __calculate_full_path__(self): From b5edfce8abbdd98f8631256feed557117e669b5b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:56:04 +0200 Subject: [PATCH 108/176] =?UTF-8?q?Scrittura=20md5=20pi=C3=B9=20pythonica.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loaih/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 62b4c32..e49ff35 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -115,12 +115,10 @@ class Build(object): # Build stays false, and we go to the next arch continue - if res.stdout: + if res.stdout and len(res.stdout.strip("\n")) > 0: # All good, the command was executed fine. - for file in res.stdout.strip('\n').split('\n'): - if self.version in open(file, 'r').read(): - print(f"Build for {self.version} found.") - self.built[arch] = True + print(f"Build for {self.version} found.") + self.built[arch] = True if self.built[arch]: print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") @@ -294,7 +292,10 @@ class Build(object): # For any built arch, find out if a file exist. if not os.path.exists(f"{item}.md5"): # Build checksum - subprocess.run(shlex.split(f"md5sum {item} > {item}.md5")) + checksum = subprocess.run(shlex.split(f"md5sum {item}"), capture_output=True, text=True, encoding='utf-8') + if checksum.stdout: + with open(f"{item}.md5", 'w+') as f: + f.write(checksum.stdout) def publish(self): From 8ded57a1548ec717a2fc7e2c45b44e5292e5bfdf Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 01:59:23 +0200 Subject: [PATCH 109/176] Forse sistemata versione appimage. --- loaih/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index e49ff35..ba132fd 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -79,7 +79,8 @@ class Build(object): def __gen_appimagefilename__(self, version, arch): """Generalize the construction of the name of the app.""" - return self.appname + "-" + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + self.appversion = self.appname + f'-{version}' + self.languagepart + self.helppart + return self.appversion + f'-{arch}.AppImage' def __calculate_full_path__(self): From be02b5076c1494f5fe2bf26257114ab16acbb1df Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 02:01:39 +0200 Subject: [PATCH 110/176] Ancora microfix a versione. --- loaih/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index ba132fd..0c57bc9 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -79,8 +79,8 @@ class Build(object): def __gen_appimagefilename__(self, version, arch): """Generalize the construction of the name of the app.""" - self.appversion = self.appname + f'-{version}' + self.languagepart + self.helppart - return self.appversion + f'-{arch}.AppImage' + self.appversion = version + self.languagepart + self.helppart + return self.appname + f'-{self.appversion}-{arch}.AppImage' def __calculate_full_path__(self): From e51b55b41eeea6b7d33204f0f38b97bccc74bb96 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 22:59:59 +0200 Subject: [PATCH 111/176] Cambiata parte finale di implementazione per creazione file generalizzati. --- loaih/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 0c57bc9..5c08954 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -50,6 +50,7 @@ class Build(object): # understood the storage_path can be changed before that phase. self.relative_path = [] self.full_path = '' + self.baseurl = '' def calculate(self): @@ -321,6 +322,7 @@ class Build(object): # Creating versions for short version and query text versions = [ self.short_version, self.branch_version ] for arch in Build.ARCHSTD: + os.chdir(self.full_path) # if the appimage for the reported arch is not found, skip to next # arch if not os.path.exists(self.appimagefilename[arch]): @@ -331,16 +333,17 @@ class Build(object): appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' zsyncfilename[arch] = appimagefilename[arch] + '.zsync' - os.chdir(self.full_path) # Create the symlink + print(f"Creating {appimagefilename[arch]} and checksums.") os.symlink(self.appimagefilename[arch], appimagefilename[arch]) # Create the checksum for the AppImage - subprocess.run(shlex.split("md5sum {item} > {item}.md5".format(item=appimagefilename[arch]))) + subprocess.run(shlex.split(f"md5sum {appimagefilename[arch]} > {appimagefilename[arch]}.md5")) # Do not continue if no zsync file is provided. if not self.updatable: continue - os.copy(self.zsyncfilename[arch], zsyncfilename[arch]) + print(f"Creating zsync file for version {version}.") + shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) From dbad601df9c33261ad3c3e210609d0936efc51c5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 23:06:49 +0200 Subject: [PATCH 112/176] Chiamo la generalizzazione. --- scripts/loaih-build | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/loaih-build b/scripts/loaih-build index 08c3892..2f830de 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -84,6 +84,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.build() obj.checksums() obj.publish() + obj.generalize_and_link() del obj if __name__ == '__main__': From b21c06038da3dd58c7580f45b9fd41f9a942d912 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 23:14:47 +0200 Subject: [PATCH 113/176] Correzione al codice per generalize. --- loaih/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index 5c08954..caa40e2 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -318,6 +318,8 @@ class Build(object): # If called with a pointed version, no generalize and link necessary. if not self.branch_version: return + appimagefilename = {} + zsyncfilename = {} # Creating versions for short version and query text versions = [ self.short_version, self.branch_version ] From f2d0630c157f9681e5a79ac37848f8488e2020cc Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 8 Apr 2022 23:26:50 +0200 Subject: [PATCH 114/176] Rimossi file di destinazione in caso di esistenza. --- loaih/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/loaih/__init__.py b/loaih/__init__.py index caa40e2..a2fdd52 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -337,6 +337,8 @@ class Build(object): # Create the symlink print(f"Creating {appimagefilename[arch]} and checksums.") + if os.path.exists(appimagefilename[arch]): + os.unlink(appimagefilename[arch]) os.symlink(self.appimagefilename[arch], appimagefilename[arch]) # Create the checksum for the AppImage subprocess.run(shlex.split(f"md5sum {appimagefilename[arch]} > {appimagefilename[arch]}.md5")) @@ -345,6 +347,8 @@ class Build(object): continue print(f"Creating zsync file for version {version}.") + if os.path.exists(zsyncfilename[arch]): + os.unlink(zsyncfilename[arch]) shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) From ca1ab61e0c0303566692819885a080d886912349 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 00:06:29 +0200 Subject: [PATCH 115/176] Sistemata modifica in place con sed. --- loaih/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index a2fdd52..7aae124 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -235,7 +235,7 @@ class Build(object): # Changing desktop file subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) - subprocess.run(shlex.split("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname), cwd=self.appimagedir) + subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) @@ -351,7 +351,7 @@ class Build(object): os.unlink(zsyncfilename[arch]) shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile - subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) + subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) def __del__(self): From ebf1e22b0abd1089f266972509139a6fa18e6f0b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 00:33:11 +0200 Subject: [PATCH 116/176] =?UTF-8?q?Cambio=20del=20default=20per=20firma:?= =?UTF-8?q?=20s=C3=AC.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/loaih-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/loaih-build b/scripts/loaih-build index 2f830de..94cda81 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -13,7 +13,7 @@ import loaih @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") @click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") -@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=True, help="Wether to sign the build. Default: sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): From e84f275f42ee4dc0c78fd266c82e6849c799fe4e Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 01:18:12 +0200 Subject: [PATCH 117/176] Cambiato sistema di ricerca delle release - ora basate su DownloadPage. --- loaih/versions.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/loaih/versions.py b/loaih/versions.py index fa0d972..201e531 100644 --- a/loaih/versions.py +++ b/loaih/versions.py @@ -6,6 +6,7 @@ from lxml import etree from packaging.version import parse as parse_version class BuildVersion(object): + DOWNLOADPAGE = "https://www.libreoffice.org/download/download/" ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" @@ -67,13 +68,23 @@ class BuildVersion(object): return { 'version': version, 'basedirurl': basedirurl } # Stable releases. - versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') - index = 1 + # Old approach - Doesn't really work because RelEng can screw order. + #versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') + #index = 1 + #if branch == 'still': + # index = -2 + #elif branch == 'fresh': + # index = -1 + #version = self.__getlatestrel(versions[index].text.strip('/')) + + # Now I'll rely on DownloadPage + versions = etree.HTML(urllib.request.urlopen(BuildVersion.DOWNLOADPAGE).read()).xpath('//span[@class="dl_version_number"]') + index = 0 if branch == 'still': - index = -2 + index = 1 elif branch == 'fresh': - index = -1 - version = self.__getlatestrel(versions[index].text.strip('/')) + index = 0 + version = self.__getlatestrel(versions[index].text) return { 'version': version, 'basedirurl': self.__getbaseurl(version) } From a3380cecc449a50bd1f36de84f2c803c417938fe Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 01:30:47 +0200 Subject: [PATCH 118/176] Creata funzione di comodo per creazione dei checksum. --- loaih/__init__.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index 7aae124..3ee146a 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -292,14 +292,17 @@ class Build(object): for arch in self.arch: for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: # For any built arch, find out if a file exist. - if not os.path.exists(f"{item}.md5"): - # Build checksum - checksum = subprocess.run(shlex.split(f"md5sum {item}"), capture_output=True, text=True, encoding='utf-8') - if checksum.stdout: - with open(f"{item}.md5", 'w+') as f: - f.write(checksum.stdout) + self.__create_checksum__(item) + def __create_checksum__(self, file): + """Internal function to create checksum file.""" + if not os.path.exists(f"{file}.md5"): + checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') + if checksum.stdout: + with open(f"{file}.md5", 'w+') as c: + c.write(checksum.stdout) + def publish(self): """Moves built versions to definitive storage.""" if all(self.built.values()): @@ -324,6 +327,10 @@ class Build(object): # Creating versions for short version and query text versions = [ self.short_version, self.branch_version ] for arch in Build.ARCHSTD: + # If already built, do not do anything. + if self.built[arch]: + continue + os.chdir(self.full_path) # if the appimage for the reported arch is not found, skip to next # arch @@ -341,7 +348,7 @@ class Build(object): os.unlink(appimagefilename[arch]) os.symlink(self.appimagefilename[arch], appimagefilename[arch]) # Create the checksum for the AppImage - subprocess.run(shlex.split(f"md5sum {appimagefilename[arch]} > {appimagefilename[arch]}.md5")) + self.__create_checksum__(appimagefilename[arch]) # Do not continue if no zsync file is provided. if not self.updatable: continue From b0d1386b49852b2f2aafef22838708e6286169c0 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 01:41:48 +0200 Subject: [PATCH 119/176] Aggiunto file per test con buildfile. Creati checksum anche per .zsync. --- loaih/__init__.py | 1 + test.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 test.yml diff --git a/loaih/__init__.py b/loaih/__init__.py index 3ee146a..dbfbd82 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -359,6 +359,7 @@ class Build(object): shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) + self.__create_checksum__(zsyncfilename[arch]) def __del__(self): diff --git a/test.yml b/test.yml new file mode 100644 index 0000000..731b596 --- /dev/null +++ b/test.yml @@ -0,0 +1,12 @@ +--- +data: + repo: /mnt/appimage + download: /var/tmp/downloads + force: no + sign: yes + +builds: + - query: fresh + language: basic + offline_help: no + portable: no From 529cd1d0c80638632756330314e2eccfe1c9e641 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 01:44:06 +0200 Subject: [PATCH 120/176] Sistemazione closing quote. --- loaih/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index dbfbd82..f5048b4 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -110,7 +110,7 @@ class Build(object): for arch in self.arch: print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}'"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') if "No such file or directory" in res.stderr: # Folder is not existent: so the version was not built From 6fb7b052fe28f20451427850bbbdbbd63f6eb8c3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 01:56:29 +0200 Subject: [PATCH 121/176] Forzata scrittura delle checksum. --- loaih/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index f5048b4..a612057 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -297,11 +297,10 @@ class Build(object): def __create_checksum__(self, file): """Internal function to create checksum file.""" - if not os.path.exists(f"{file}.md5"): - checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') - if checksum.stdout: - with open(f"{file}.md5", 'w+') as c: - c.write(checksum.stdout) + checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') + if checksum.stdout: + with open(f"{file}.md5", 'w') as c: + c.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" From 380fd7029e6314f598dab0ffd631e816d70831c4 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Apr 2022 02:02:04 +0200 Subject: [PATCH 122/176] Forzato build in test.yml. --- test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.yml b/test.yml index 731b596..faf4464 100644 --- a/test.yml +++ b/test.yml @@ -2,7 +2,7 @@ data: repo: /mnt/appimage download: /var/tmp/downloads - force: no + force: yes sign: yes builds: From 638706ce2f1a739a570ea3334a1cc03a5b58650f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 24 Apr 2022 22:38:40 +0200 Subject: [PATCH 123/176] Tentative prerelease support. Unfortunately requires the rewrite of most of the scripts, as it might provide multiple versions. --- loaih/versions.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/loaih/versions.py b/loaih/versions.py index 201e531..dc90f17 100644 --- a/loaih/versions.py +++ b/loaih/versions.py @@ -30,8 +30,18 @@ class BuildVersion(object): else: # String self.versions. a = self.__getbranchrel(self.query) - self.version = a['version'] - self.basedirurl = a['basedirurl'] + + if isinstance(a, list): + # If it's a list, a number of releases are provided. + self.version = [] + self.basedirurl = [] + for i in range(len(a)): + self.version[i] = self.__getlatestrel(a[i]['version']) + self.basedirurl[i] = a[i]['basedirurl'] + + else: + self.version = a['version'] + self.basedirurl = a['basedirurl'] def __getlatestrel(self, basever): """Search in downloadarchive for the latest version matching baseversion.""" @@ -62,10 +72,18 @@ class BuildVersion(object): return { 'version': version + '-' + dailyversion, 'basedirurl': basedirurl } if branch == 'prerelease': - version = etree.HTML(urllib.request.urlopen(BuildVersion.PRERELEASE).read()).xpath('//td/a')[1].text.split('_')[1] + # Also here, we'll rely on DownloadPage. Whenever a prerelease version is cited in the page. + version = etree.HTML(urllib.request.urlopen(BuildVersion.DOWNLOADPAGE).read()).xpath('//p[@class="lead_libre"][last()]/following-sibling::ul[last()]/li/a/text()') + retval = [] basedirurl = { u'x86': '-', u'x86_64': BuildVersion.PRERELEASE } + + if len(version) == 0: + return retval - return { 'version': version, 'basedirurl': basedirurl } + for v in version: + retval.append({ 'version': v, 'basedirurl': basedirurl }) + + return retval # Stable releases. # Old approach - Doesn't really work because RelEng can screw order. From 0fae769ec0f4640d9a7d522e2d6a4b7834244f35 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 02:14:34 +0200 Subject: [PATCH 124/176] =?UTF-8?q?versions=20rivisto=20e=20quasi=20riscri?= =?UTF-8?q?tto.=20getversion=20da=20rivedere,=20perch=C3=A9=20non=20era=20?= =?UTF-8?q?stato=20previsto=20caso=20di=20multiple=20versioni=20ritornate.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loaih/versions.py | 242 ++++++++++++++++++++++----------------- scripts/loaih-getversion | 22 ++-- 2 files changed, 154 insertions(+), 110 deletions(-) diff --git a/loaih/versions.py b/loaih/versions.py index dc90f17..ade3fbc 100644 --- a/loaih/versions.py +++ b/loaih/versions.py @@ -4,119 +4,157 @@ import urllib.request from lxml import etree from packaging.version import parse as parse_version +import datetime -class BuildVersion(object): +class Definitions(object): DOWNLOADPAGE = "https://www.libreoffice.org/download/download/" ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" PRERELEASE = "https://dev-builds.libreoffice.org/pre-releases/deb/x86_64/" + SELECTORS = { + 'still': { + 'URL': DOWNLOADPAGE, + 'xpath': '(//span[@class="dl_version_number"])[last()]/text()' + }, + 'fresh': { + 'URL': DOWNLOADPAGE, + 'xpath': '(//span[@class="dl_version_number"])[1]/text()' + }, + 'prerelease': { + 'URL': DOWNLOADPAGE, + 'xpath': '//p[@class="lead_libre"][last()]/following-sibling::ul[last()]/li/a/text()' + }, + 'daily': { + 'URL': DAILY, + 'xpath': '//td/a' + } + } + +class Base(object): + # Class for static methods which might be useful even outside the build + # scripts. + + @staticmethod + def dailyver(date = datetime.datetime.today()): + """Returns versions present on the latest daily build.""" + url = Base.dailyurl(date)['x86_64'] + # Since it is possible that a daily build is not yet provided... + if url == '-': + return [] + + # Rerun the page parsing, this time to find out the versions built + + b = etree.HTML(urllib.request.urlopen(Base.dailyurl(date)['x86_64']).read()).xpath("//td/a[contains(text(), '_deb.tar.gz')]/text()") + # This should have returned the main package for a version, but can + # have returned multiple ones, so let's treat it as a list + return [ x.split('_')[1] for x in b ] + + @staticmethod + def dailyurl(date = datetime.datetime.today()): + """Returns the URL for the latest valid daily build.""" + # As per other parts of the build, we need to maintain an URL also for + # x86 versions that it isn't really provided. + # As such, the return value must be a dictionary + + # Get the anchor for today's builds + a = etree.HTML(urllib.request.urlopen(Definitions.DAILY).read()).xpath("//td/a[contains(text(), '" + date.strftime('%Y-%m-%d') + "')]/text()") + if len(a) == 0: + # No results found, no version found, let's return a + return { 'x86': '-', 'x86_64': '-' } + + # On the contrary, more than a version is found. let's order the + # list and get the latest item + return { 'x86': '-', 'x86_64': Definitions.SELECTORS['daily']['URL'] + sorted(a)[-1] } + + @staticmethod + def namedver(query): + """Gets the version for a specific named version.""" + + if query == 'daily' or query == 'yesterday': + # Daily needs double parsing for the same result to apply. + # We first select today's build anchor: + date = datetime.datetime.today() + if query == 'yesterday': + # Use yesterdays' date for testing purposes. + date += datetime.timedelta(days=-1) + return Base.dailyver(date) + + # In case the query isn't for daily + return etree.HTML(urllib.request.urlopen(Definitions.SELECTORS[query]['URL']).read()).xpath(Definitions.SELECTORS[query]['xpath']) + + @staticmethod + def fullversion(version): + """Get latest full version from Archive based on partial version.""" + versionlist = etree.HTML(urllib.request.urlopen(Definitions.ARCHIVE).read()).xpath("//td/a[starts-with(text(), '" + version + "')]/text()") + cleanlist = sorted([ x.strip('/') for x in versionlist ]) + + # Sorting, then returning the last version + return cleanlist[-1] + + @staticmethod + def urlfromqueryandver(query, version): + """Returns the fetching URL based on the queried version and the numeric version of it.""" + # This has the purpose to simplify and explain how the releases are + # layed out. + + # If the query tells about daily or 'yesterday' (for testing purposes), + # we might ignore versions and return the value coming from dailyurl: + if query == 'daily': + return Base.dailyurl() + if query == 'yesterday': + date = datetime.datetime.today() + datetime.timedelta(days=-1) + return Base.dailyurl(date) + + # All other versions will be taken from Archive, as such we need a full + # version. + + # If the version has only 2 points in it (or splits into three parts by '.'), that's not a full version and we will call the getlatestver() function + fullversion = version + if len(version.split('.')) == 3: + fullversion = Base.fullversion(version) + + # So the final URL is the Archive one, plus the full versions, plus a + # final '/deb/' - and an arch subfolder + baseurl = Definitions.ARCHIVE + fullversion + '/deb/' + retval = {} + + # x86 binaries are not anymore offered after 6.3.0. + if parse_version(version) < parse_version('6.3.0'): + retval['x86'] = baseurl + 'x86/' + else: + retval['x86'] = '-' + + retval['x86_64'] = baseurl + 'x86_64/' + + return retval + + +class Build(object): + def __init__(self, query): + """Should simplify the single builded version.""" self.query = query self.version = '' self.basedirurl = {} - # Parsing the query input. - if '.' in self.query: - # Numbered self.version. Let's check it is a 4 dotted release - if len(self.query.split('.')) == 4: - self.version = self.query - else: - # If not 4 dotted, let's search for the 4 dotted version - self.version = self.__getlatestrel(self.query) - - self.basedirurl = self.__getbaseurl(self.version) - else: - # String self.versions. - a = self.__getbranchrel(self.query) + # For simplification process, let's before see if there's a named query + # (it shouldn't) + if not '.' in self.query: + # Named version. Let's get the result of the query. + a = Base.namedver(self.query) - if isinstance(a, list): - # If it's a list, a number of releases are provided. - self.version = [] - self.basedirurl = [] - for i in range(len(a)): - self.version[i] = self.__getlatestrel(a[i]['version']) - self.basedirurl[i] = a[i]['basedirurl'] - - else: - self.version = a['version'] - self.basedirurl = a['basedirurl'] - - def __getlatestrel(self, basever): - """Search in downloadarchive for the latest version matching baseversion.""" - versionlist = etree.HTML(urllib.request.urlopen(BuildVersion.ARCHIVE).read()).xpath('//td/a') - # Getting a more polished matching list - cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) - - # Sorting, then returning the last version - return sorted(cleanlist)[-1] - - def __getbranchrel(self, branch): - """Based on branch names, get the release number.""" - basedirurl = {} - version = '' - if branch == 'daily': - # The daily builds can be mostly distinguished by the day of build - # (official version is constant. - - # The last built version is the next-to-last version [-2] on the page. - fulldailypath = etree.HTML(urllib.request.urlopen(BuildVersion.DAILY).read()).xpath('//td/a')[-2].text - dailyversion = fulldailypath.split('_')[0].replace('-', '') - version - newurl = str.join('/', [ BuildVersion.DAILY, fulldailypath, '' ]) - - basedirurl = { u'x86_64': newurl, u'x86': '-' } - version = etree.HTML(urllib.request.urlopen(newurl).read()).xpath('//td/a')[1].text.split('_')[1] - - return { 'version': version + '-' + dailyversion, 'basedirurl': basedirurl } - - if branch == 'prerelease': - # Also here, we'll rely on DownloadPage. Whenever a prerelease version is cited in the page. - version = etree.HTML(urllib.request.urlopen(BuildVersion.DOWNLOADPAGE).read()).xpath('//p[@class="lead_libre"][last()]/following-sibling::ul[last()]/li/a/text()') - retval = [] - basedirurl = { u'x86': '-', u'x86_64': BuildVersion.PRERELEASE } - - if len(version) == 0: - return retval - - for v in version: - retval.append({ 'version': v, 'basedirurl': basedirurl }) - - return retval - - # Stable releases. - # Old approach - Doesn't really work because RelEng can screw order. - #versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') - #index = 1 - #if branch == 'still': - # index = -2 - #elif branch == 'fresh': - # index = -1 - #version = self.__getlatestrel(versions[index].text.strip('/')) - - # Now I'll rely on DownloadPage - versions = etree.HTML(urllib.request.urlopen(BuildVersion.DOWNLOADPAGE).read()).xpath('//span[@class="dl_version_number"]') - index = 0 - if branch == 'still': - index = 1 - elif branch == 'fresh': - index = 0 - version = self.__getlatestrel(versions[index].text) - - return { 'version': version, 'basedirurl': self.__getbaseurl(version) } - - def __getbaseurl(self, version): - """Returns the links based on the numeric version.""" - basedirurl = {} - url = BuildVersion.ARCHIVE + '/' + version + '/deb/' - - # x86 binaries are not anymore offered after 6.3.0. - if parse_version(version) < parse_version('6.3.0'): - basedirurl[u'x86'] = url + 'x86/' - else: - basedirurl[u'x86'] = '-' + if len(a) != 1: + raise Exception("Queried version for build does not return a single result. Please unfold manually. Exiting.") - basedirurl[u'x86_64'] = url + 'x86_64/' - - return basedirurl + # So if it is here, the version is one. + self.query = a[0] + + if len(self.query.split('.')) == 4: + self.version = self.query + else: + # If not 4 dotted, let's search for the 4 dotted version + self.version = Base.fullversion(self.query) + + self.basedirurl = Base.urlfromqueryandver(self.query, self.version) diff --git a/scripts/loaih-getversion b/scripts/loaih-getversion index 59d03ae..e9d2c04 100644 --- a/scripts/loaih-getversion +++ b/scripts/loaih-getversion @@ -2,21 +2,27 @@ # encoding: utf-8 import click -from loaih.versions import BuildVersion +import loaih.versions as versions import re, sys, json @click.command() @click.option('-o', '--output', default = 'rundeck', type=click.Choice(['rundeck', 'json', 'text' ], case_sensitive=False), help="Output format, defaulting to Rundeck Key/Value data format. Options: rundeck,json,text") @click.argument('query') def getversion(query, output): - b = BuildVersion(query) + b = [] + if '.' in query: + b.append(versions.Build(query)) + else: + # In case of names, we might want to loop for versions. So we before + # get the versions available, based on query. + vers = versions.Base.namedver(query) + if len(vers) != 1: + for v in vers: + b.append(versions.Build(v)) + else: + b = versions.Build(vers[0]) - if output.lower() == 'rundeck': - print("""RUNDECK:DATA: query = {query} -RUNDECK:DATA: version = {version} -RUNDECK:DATA: x86 = {x86_url} -RUNDECK:DATA: x86_64 = {x86_64_url}""".format(query = query, version = b.version, x86_url = b.basedirurl['x86'], x86_64_url = b.basedirurl['x86_64'])) - elif output.lower() == 'json': + if output.lower() == 'json': output = { 'query': query, 'version': b.version, From c30407a244adbe37a87415b2c5b23b928949cd43 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 16:58:00 +0200 Subject: [PATCH 125/176] Refactor parziale dello script. Refactor della parte getversion. --- scripts/loaih-build => loaih/script.py | 36 ++++++++-- loaih/versions.py | 99 +++++++++++++++++--------- scripts/loaih-getversion | 40 ----------- setup.py | 6 +- 4 files changed, 104 insertions(+), 77 deletions(-) rename scripts/loaih-build => loaih/script.py (82%) delete mode 100644 scripts/loaih-getversion diff --git a/scripts/loaih-build b/loaih/script.py similarity index 82% rename from scripts/loaih-build rename to loaih/script.py index 94cda81..5e8499f 100644 --- a/scripts/loaih-build +++ b/loaih/script.py @@ -4,8 +4,39 @@ import click import yaml import loaih +import re, sys, json -@click.command() +@click.group() +def cli(): + pass + +@cli.command() +@click.option('-j', '--json', 'jsonout', default=False, is_flag=True, help="Output format in json.") +@click.argument('query') +def getversion(query, jsonout): + b = [] + queries = [] + if ',' in query: + queries.extend(query.split(',')) + else: + queries.append(query) + + for q in queries: + if '.' in q: + # Numbered version. It is safe to send it to Build. + b.append(loaih.versions.Build(q)) + else: + # Named version. For safety, we call a helper method for a collection + b.extend(loaih.versions.Base.collectedbuilds(q)) + + if len(b) > 0: + if jsonout: + click.echo(json.dumps([x.todict() for x in b])) + else: + for v in b: + click.echo(v) + +@cli.command() @click.option('-a', '--arch', 'arch', type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), default='all', help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") @click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") @click.option('-d', '--download-path', 'download_path', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") @@ -86,6 +117,3 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.publish() obj.generalize_and_link() del obj - -if __name__ == '__main__': - build() diff --git a/loaih/versions.py b/loaih/versions.py index ade3fbc..3192438 100644 --- a/loaih/versions.py +++ b/loaih/versions.py @@ -36,21 +36,6 @@ class Base(object): # Class for static methods which might be useful even outside the build # scripts. - @staticmethod - def dailyver(date = datetime.datetime.today()): - """Returns versions present on the latest daily build.""" - url = Base.dailyurl(date)['x86_64'] - # Since it is possible that a daily build is not yet provided... - if url == '-': - return [] - - # Rerun the page parsing, this time to find out the versions built - - b = etree.HTML(urllib.request.urlopen(Base.dailyurl(date)['x86_64']).read()).xpath("//td/a[contains(text(), '_deb.tar.gz')]/text()") - # This should have returned the main package for a version, but can - # have returned multiple ones, so let's treat it as a list - return [ x.split('_')[1] for x in b ] - @staticmethod def dailyurl(date = datetime.datetime.today()): """Returns the URL for the latest valid daily build.""" @@ -68,6 +53,20 @@ class Base(object): # list and get the latest item return { 'x86': '-', 'x86_64': Definitions.SELECTORS['daily']['URL'] + sorted(a)[-1] } + @staticmethod + def dailyver(date = datetime.datetime.today()): + """Returns versions present on the latest daily build.""" + url = Base.dailyurl(date)['x86_64'] + # If no daily releases has been provided yet, return empty + if url == '-': + return [] + + # Rerun the page parsing, this time to find out the versions built + b = etree.HTML(urllib.request.urlopen(url).read()).xpath("//td/a[contains(text(), '_deb.tar.gz')]/text()") + # This should have returned the main package for a version, but can + # have returned multiple ones, so let's treat it as a list + return [ x.split('_')[1] for x in b ] + @staticmethod def namedver(query): """Gets the version for a specific named version.""" @@ -87,7 +86,7 @@ class Base(object): @staticmethod def fullversion(version): """Get latest full version from Archive based on partial version.""" - versionlist = etree.HTML(urllib.request.urlopen(Definitions.ARCHIVE).read()).xpath("//td/a[starts-with(text(), '" + version + "')]/text()") + versionlist = etree.HTML(urllib.request.urlopen(Definitions.ARCHIVE).read()).xpath(f"//td/a[starts-with(text(), '{version}')]/text()") cleanlist = sorted([ x.strip('/') for x in versionlist ]) # Sorting, then returning the last version @@ -112,7 +111,7 @@ class Base(object): # If the version has only 2 points in it (or splits into three parts by '.'), that's not a full version and we will call the getlatestver() function fullversion = version - if len(version.split('.')) == 3: + if len(version.split('.')) <= 3: fullversion = Base.fullversion(version) # So the final URL is the Archive one, plus the full versions, plus a @@ -130,31 +129,67 @@ class Base(object): return retval + @staticmethod + def collectedbuilds(query): + """Creates a list of Builds based on each namedver found.""" + retval = [] + a = Base.namedver(query) + if isinstance(a, list) and len(a) > 1: + retval = [ Build(query, version) for version in Base.namedver(query) ] + else: + retval.append(Build(query)) + + return retval + class Build(object): - def __init__(self, query): + def __init__(self, query, version = None): """Should simplify the single builded version.""" self.query = query self.version = '' - self.basedirurl = {} + self.basedirurl = { 'x86': '-', 'x86_64': '-' } - # For simplification process, let's before see if there's a named query - # (it shouldn't) if not '.' in self.query: - # Named version. Let's get the result of the query. - a = Base.namedver(self.query) + # Named version. + # Let's check if a specific version was requested. + if version: + self.version = version + else: + # In case it was not requested, we will carry on the generic + # namedver() query. + # If the results are more than one, we'll take the latest (since we are requested to provide a single build). + a = Base.namedver(self.query) + + if isinstance(a, list) and len(a) == 0: + # No results from the query - let's return default values + return - if len(a) != 1: - raise Exception("Queried version for build does not return a single result. Please unfold manually. Exiting.") - - # So if it is here, the version is one. - self.query = a[0] - - if len(self.query.split('.')) == 4: - self.version = self.query + if len(a) == 1: + # version is a single one. + self.version = a[0] + else: + # In this case, we will select the latest release. + self.version = sorted(a)[-1] else: + # In case of numbered queries, put it as initial version + self.version = self.query + + if len(self.version.split('.')) < 4: # If not 4 dotted, let's search for the 4 dotted version - self.version = Base.fullversion(self.query) + self.version = Base.fullversion(self.version) self.basedirurl = Base.urlfromqueryandver(self.query, self.version) + + def todict(self): + return { + 'query': self.query, + 'version': self.version, + 'basedirurl': self.basedirurl + } + + def __str__(self): + return f"""query: {self.query} +version: {self.version} +x86: {self.basedirurl['x86']} +x86_64: {self.basedirurl['x86_64']}""" diff --git a/scripts/loaih-getversion b/scripts/loaih-getversion deleted file mode 100644 index e9d2c04..0000000 --- a/scripts/loaih-getversion +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import click -import loaih.versions as versions -import re, sys, json - -@click.command() -@click.option('-o', '--output', default = 'rundeck', type=click.Choice(['rundeck', 'json', 'text' ], case_sensitive=False), help="Output format, defaulting to Rundeck Key/Value data format. Options: rundeck,json,text") -@click.argument('query') -def getversion(query, output): - b = [] - if '.' in query: - b.append(versions.Build(query)) - else: - # In case of names, we might want to loop for versions. So we before - # get the versions available, based on query. - vers = versions.Base.namedver(query) - if len(vers) != 1: - for v in vers: - b.append(versions.Build(v)) - else: - b = versions.Build(vers[0]) - - if output.lower() == 'json': - output = { - 'query': query, - 'version': b.version, - 'basedirurl': b.basedirurl - } - print(json.dumps(output)) - else: - print("""query: {query} -version: {version} -x86: {x86_url} -x86_64: {x86_64_url}""".format(query = query, version = b.version, x86_url = b.basedirurl['x86'], x86_64_url = b.basedirurl['x86_64'])) - - -if __name__ == '__main__': - getversion() diff --git a/setup.py b/setup.py index 4ee9309..e4ad69f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,11 @@ setup( author="Emiliano Vavassori", author_email="syntaxerrormmm@libreoffice.org", packages=find_packages(exclude=['contrib', 'docs', 'tests']), - scripts=[ 'scripts/loaih-getversion', 'scripts/loaih-build' ], + entry_points={ + 'console_scripts': [ + 'loaih = loaih.script:cli', + ], + }, install_requires=[ 'click', ], license='MIT', url='https://git.libreitalia.org/LibreItalia/loappimage-helpers/', From 15369c08957dd0249b5cd92e203a653b9afed835 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 17:46:46 +0200 Subject: [PATCH 126/176] Rinominati file all'interno del progetto. --- loaih/__init__.py | 534 ++++++++++++++++------------------------------ loaih/build.py | 367 +++++++++++++++++++++++++++++++ loaih/script.py | 4 +- loaih/versions.py | 195 ----------------- 4 files changed, 550 insertions(+), 550 deletions(-) create mode 100644 loaih/build.py delete mode 100644 loaih/versions.py diff --git a/loaih/__init__.py b/loaih/__init__.py index a612057..b102568 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -1,367 +1,195 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python +# encoding: utf-8 import urllib.request -import loaih.versions as versions from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re, shlex +from packaging.version import parse as parse_version +import datetime -class Build(object): - LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] - LANGBASIC = [ 'en-GB' ] - ARCHSTD = [ u'x86', u'x86_64' ] +class Definitions(object): + DOWNLOADPAGE = "https://www.libreoffice.org/download/download/" + ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" + RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" + DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" + PRERELEASE = "https://dev-builds.libreoffice.org/pre-releases/deb/x86_64/" - def __init__(self, query, arch): - """Build all versions that can be found in the indicated repo.""" + SELECTORS = { + 'still': { + 'URL': DOWNLOADPAGE, + 'xpath': '(//span[@class="dl_version_number"])[last()]/text()' + }, + 'fresh': { + 'URL': DOWNLOADPAGE, + 'xpath': '(//span[@class="dl_version_number"])[1]/text()' + }, + 'prerelease': { + 'URL': DOWNLOADPAGE, + 'xpath': '//p[@class="lead_libre"][last()]/following-sibling::ul[last()]/li/a/text()' + }, + 'daily': { + 'URL': DAILY, + 'xpath': '//td/a' + } + } + +class Base(object): + # Class for static methods which might be useful even outside the build + # scripts. + + @staticmethod + def dailyurl(date = datetime.datetime.today()): + """Returns the URL for the latest valid daily build.""" + # As per other parts of the build, we need to maintain an URL also for + # x86 versions that it isn't really provided. + # As such, the return value must be a dictionary + + # Get the anchor for today's builds + a = etree.HTML(urllib.request.urlopen(Definitions.DAILY).read()).xpath("//td/a[contains(text(), '" + date.strftime('%Y-%m-%d') + "')]/text()") + if len(a) == 0: + # No results found, no version found, let's return a + return { 'x86': '-', 'x86_64': '-' } + + # On the contrary, more than a version is found. let's order the + # list and get the latest item + return { 'x86': '-', 'x86_64': Definitions.SELECTORS['daily']['URL'] + sorted(a)[-1] } + + @staticmethod + def dailyver(date = datetime.datetime.today()): + """Returns versions present on the latest daily build.""" + url = Base.dailyurl(date)['x86_64'] + # If no daily releases has been provided yet, return empty + if url == '-': + return [] + + # Rerun the page parsing, this time to find out the versions built + b = etree.HTML(urllib.request.urlopen(url).read()).xpath("//td/a[contains(text(), '_deb.tar.gz')]/text()") + # This should have returned the main package for a version, but can + # have returned multiple ones, so let's treat it as a list + return [ x.split('_')[1] for x in b ] + + @staticmethod + def namedver(query): + """Gets the version for a specific named version.""" + + if query == 'daily' or query == 'yesterday': + # Daily needs double parsing for the same result to apply. + # We first select today's build anchor: + date = datetime.datetime.today() + if query == 'yesterday': + # Use yesterdays' date for testing purposes. + date += datetime.timedelta(days=-1) + return Base.dailyver(date) + + # In case the query isn't for daily + return etree.HTML(urllib.request.urlopen(Definitions.SELECTORS[query]['URL']).read()).xpath(Definitions.SELECTORS[query]['xpath']) + + @staticmethod + def fullversion(version): + """Get latest full version from Archive based on partial version.""" + versionlist = etree.HTML(urllib.request.urlopen(Definitions.ARCHIVE).read()).xpath(f"//td/a[starts-with(text(), '{version}')]/text()") + cleanlist = sorted([ x.strip('/') for x in versionlist ]) + + # Sorting, then returning the last version + return cleanlist[-1] + + @staticmethod + def urlfromqueryandver(query, version): + """Returns the fetching URL based on the queried version and the numeric version of it.""" + # This has the purpose to simplify and explain how the releases are + # layed out. + + # If the query tells about daily or 'yesterday' (for testing purposes), + # we might ignore versions and return the value coming from dailyurl: + if query == 'daily': + return Base.dailyurl() + if query == 'yesterday': + date = datetime.datetime.today() + datetime.timedelta(days=-1) + return Base.dailyurl(date) + + # All other versions will be taken from Archive, as such we need a full + # version. + + # If the version has only 2 points in it (or splits into three parts by '.'), that's not a full version and we will call the getlatestver() function + fullversion = version + if len(version.split('.')) <= 3: + fullversion = Base.fullversion(version) + + # So the final URL is the Archive one, plus the full versions, plus a + # final '/deb/' - and an arch subfolder + baseurl = Definitions.ARCHIVE + fullversion + '/deb/' + retval = {} + + # x86 binaries are not anymore offered after 6.3.0. + if parse_version(version) < parse_version('6.3.0'): + retval['x86'] = baseurl + 'x86/' + else: + retval['x86'] = '-' + + retval['x86_64'] = baseurl + 'x86_64/' + + return retval + + @staticmethod + def collectedbuilds(query): + """Creates a list of Builds based on each namedver found.""" + retval = [] + a = Base.namedver(query) + if isinstance(a, list) and len(a) > 1: + retval = [ Build(query, version) for version in Base.namedver(query) ] + else: + retval.append(RemoteBuild(query)) + + return retval + + +class RemoteBuild(object): + + def __init__(self, query, version = None): + """Should simplify the single builded version.""" self.query = query - self.arch = arch - - # Getting versions and so on - v = versions.BuildVersion(self.query) - self.version = v.version - print(f"Debug {self.version}") - self.short_version = str.join('.', self.version.split('.')[0:2]) - self.branch_version = None + self.version = '' + self.basedirurl = { 'x86': '-', 'x86_64': '-' } + if not '.' in self.query: - self.branch_version = self.query - self.url = v.basedirurl - - # Other default values - self.language = 'basic' - self.offline_help = False - self.portable = False - self.updatable = True - self.sign = True - self.storage_path = '/mnt/appimage' - self.download_path = '/var/tmp/downloads' - - # Specific build version - self.appversion = '' - self.appimagefilename = {} - self.zsyncfilename = {} - - # Creating a tempfile - self.builddir = tempfile.mkdtemp() - self.tarballs = {} - self.built = { u'x86': False, u'x86_64': False } - - # Preparing the default for the relative path on the storage for - # different versions. - # The path will evaluated as part of the check() function, as it is - # understood the storage_path can be changed before that phase. - self.relative_path = [] - self.full_path = '' - self.baseurl = '' - - - def calculate(self): - """Calculate exclusions and other variables.""" - # AppName - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' - - # Calculating languagepart - self.languagepart = "." - if ',' in self.language: - self.languagepart += self.language.replace(',', '-') - else: - self.languagepart += self.language - - # Calculating help part - self.helppart = '.help' if self.offline_help else '' - - # Building the required names - for arch in Build.ARCHSTD: - self.appimagefilename[arch] = self.__gen_appimagefilename__(self.version, arch) - self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' - - # Mandate to the private function to calculate the full_path available - # for the storage and the checks. - self.__calculate_full_path__() - - - def __gen_appimagefilename__(self, version, arch): - """Generalize the construction of the name of the app.""" - self.appversion = version + self.languagepart + self.helppart - return self.appname + f'-{self.appversion}-{arch}.AppImage' - - - def __calculate_full_path__(self): - """Calculate relative path of the build, based on internal other variables.""" - if len(self.relative_path) == 0: - if self.query == 'daily': - self.relative_path.append('daily') - elif self.query == 'primageerelease': - self.relative_path.append('prerelease') - - # Not the same check, an additional one - if self.portable: - self.relative_path.append('portable') - - fullpath_arr = self.storage_path.split('/') - # Joining relative path only if it is not null - if len(self.relative_path) > 0: - fullpath_arr.extend(self.relative_path) - self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) - - - def check(self): - """Checking if the requested AppImage has been already built.""" - if not len(self.appimagefilename) == 2: - self.calculate() - - for arch in self.arch: - print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') - - if "No such file or directory" in res.stderr: - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue - - if res.stdout and len(res.stdout.strip("\n")) > 0: - # All good, the command was executed fine. - print(f"Build for {self.version} found.") - self.built[arch] = True - - if self.built[arch]: - print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") - - - def download(self): - """Downloads the contents of the URL as it was a folder.""" - print(f"Started downloads for {self.version}. Please wait.") - for arch in self.arch: - # Checking if a valid path has been provided - if self.url[arch] == '-': - print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.") - # Faking already built it so to skip other checks. - self.built[arch] = True - continue - - if self.built[arch]: - print(f"A build for {arch} was already found. Skipping specific packages.") - continue - - # Identifying downloads - contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] - tarballs = self.tarballs[arch] - maintarball = tarballs[0] - - # Create and change directory to the download location - os.makedirs(self.download_path, exist_ok = True) - os.chdir(self.download_path) - for archive in tarballs: - # If the archive is already there, do not do anything. - if os.path.exists(archive): - continue - - # Download the archive - try: - urllib.request.urlretrieve(self.url[arch] + archive, archive) - except: - print(f"Failed to download {archive}.") - - print(f"Finished downloads for {self.version}.") - - def build(self): - """Building all the versions.""" - - for arch in self.arch: - if self.built[arch]: - # Already built for arch or path not available. User has already been warned. - continue - - # Preparation tasks - self.appnamedir = os.path.join(self.builddir, self.appname) - os.makedirs(self.appnamedir, exist_ok=True) - # And then cd to the appname folder. - os.chdir(self.appnamedir) - # Download appimagetool from github - appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage" - urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') - os.chmod('appimagetool', 0o755) - - # Build the requested version. - self.__unpackbuild__(arch) - - - def __unpackbuild__(self, arch): - # We start by filtering out tarballs from the list - buildtarballs = [ self.tarballs[arch][0] ] - - # Let's process standard languages and append results to the - # buildtarball - if self.language == 'basic': - if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'pack_en-GB' in x ]) + # Named version. + # Let's check if a specific version was requested. + if version: + self.version = version else: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) - elif self.language == 'standard': - for lang in Build.LANGSTD: - if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack_' + lang) in x ]) - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack_' + lang) in x ]) - elif self.language == 'full': - if self.offline_help: - # We need also all help. Let's replace buildtarball with the - # whole bunch - buildtarballs = self.tarballs[arch] - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) - else: - # Looping for each language in self.language - for lang in self.language.split(","): - if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) + # In case it was not requested, we will carry on the generic + # namedver() query. + # If the results are more than one, we'll take the latest (since we are requested to provide a single build). + a = Base.namedver(self.query) - os.chdir(self.appnamedir) + if isinstance(a, list) and len(a) == 0: + # No results from the query - let's return default values + return - # Unpacking the tarballs - for archive in buildtarballs: - subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + if len(a) == 1: + # version is a single one. + self.version = a[0] + else: + # In this case, we will select the latest release. + self.version = sorted(a)[-1] + else: + # In case of numbered queries, put it as initial version + self.version = self.query - # create appimagedir - self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') - os.makedirs(self.appimagedir, exist_ok = True) - - # At this point, let's decompress the deb packages - subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) - - if self.portable: - subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) - - # Changing desktop file - subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) - subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) - - subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) - - # Find the name of the binary called in the desktop file. - binaryname = '' - with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: - a = d.readlines() - for line in a: - if re.match(r'^Exec', line): - binaryname = line.split('=')[-1].split(' ')[0] - # Esci al primo match - break - #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') - #binaryname = binary_exec.stdout.strip("\n") - - bindir=os.path.join(self.appimagedir, 'usr', 'bin') - os.makedirs(bindir, exist_ok = True) - subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) - - # Download AppRun from github - apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" - dest = os.path.join(self.appimagedir, 'AppRun') - urllib.request.urlretrieve(apprunurl, dest) - os.chmod(dest, 0o755) - - # Dealing with extra options - buildopts = [] - if self.sign: - buildopts.append('--sign') - - # adding zsync build if updatable - if self.updatable: - buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'") - - buildopts_str = str.join(' ', buildopts) - # Build the number-specific build - subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) + if len(self.version.split('.')) < 4: + # If not 4 dotted, let's search for the 4 dotted version + self.version = Base.fullversion(self.version) - print(f"Built AppImage version {self.appversion}") + self.basedirurl = Base.urlfromqueryandver(self.query, self.version) - # Cleanup phase, before new run. - for deb in glob.glob(self.appnamedir + '/*.deb'): - os.remove(deb) - subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) + def todict(self): + return { + 'query': self.query, + 'version': self.version, + 'basedirurl': self.basedirurl + } - - def checksums(self): - """Create checksums of the built versions.""" - # Skip checksum if initally the build was already found in the storage directory - if all(self.built.values()): - return - - os.chdir(self.appnamedir) - for arch in self.arch: - for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - # For any built arch, find out if a file exist. - self.__create_checksum__(item) - - - def __create_checksum__(self, file): - """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') - if checksum.stdout: - with open(f"{file}.md5", 'w') as c: - c.write(checksum.stdout) - - def publish(self): - """Moves built versions to definitive storage.""" - if all(self.built.values()): - # All files are already present in the full_path - return - - os.chdir(self.appnamedir) - # Forcing creation of subfolders, in case there is a new build - os.makedirs(self.full_path, exist_ok = True) - for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - - - def generalize_and_link(self): - """Creates the needed generalized files if needed.""" - # If called with a pointed version, no generalize and link necessary. - if not self.branch_version: - return - appimagefilename = {} - zsyncfilename = {} - - # Creating versions for short version and query text - versions = [ self.short_version, self.branch_version ] - for arch in Build.ARCHSTD: - # If already built, do not do anything. - if self.built[arch]: - continue - - os.chdir(self.full_path) - # if the appimage for the reported arch is not found, skip to next - # arch - if not os.path.exists(self.appimagefilename[arch]): - continue - - # Doing it both for short_name and for branchname - for version in versions: - appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' - zsyncfilename[arch] = appimagefilename[arch] + '.zsync' - - # Create the symlink - print(f"Creating {appimagefilename[arch]} and checksums.") - if os.path.exists(appimagefilename[arch]): - os.unlink(appimagefilename[arch]) - os.symlink(self.appimagefilename[arch], appimagefilename[arch]) - # Create the checksum for the AppImage - self.__create_checksum__(appimagefilename[arch]) - # Do not continue if no zsync file is provided. - if not self.updatable: - continue - - print(f"Creating zsync file for version {version}.") - if os.path.exists(zsyncfilename[arch]): - os.unlink(zsyncfilename[arch]) - shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) - # Editing the zsyncfile - subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) - self.__create_checksum__(zsyncfilename[arch]) - - - def __del__(self): - """Destructor""" - # Cleaning up build directory - shutil.rmtree(self.builddir) + def __str__(self): + return f"""query: {self.query} +version: {self.version} +x86: {self.basedirurl['x86']} +x86_64: {self.basedirurl['x86_64']}""" diff --git a/loaih/build.py b/loaih/build.py new file mode 100644 index 0000000..59e648a --- /dev/null +++ b/loaih/build.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 + +import urllib.request +import loaih +from lxml import etree +import tempfile, os, sys, glob, subprocess, shutil, re, shlex + +class Build(object): + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] + LANGBASIC = [ 'en-GB' ] + ARCHSTD = [ u'x86', u'x86_64' ] + + def __init__(self, query, arch): + """Build all versions that can be found in the indicated repo.""" + self.query = query + self.arch = arch + + # Getting versions and so on + v = versions.BuildVersion(self.query) + self.version = v.version + print(f"Debug {self.version}") + self.short_version = str.join('.', self.version.split('.')[0:2]) + self.branch_version = None + if not '.' in self.query: + self.branch_version = self.query + self.url = v.basedirurl + + # Other default values + self.language = 'basic' + self.offline_help = False + self.portable = False + self.updatable = True + self.sign = True + self.storage_path = '/mnt/appimage' + self.download_path = '/var/tmp/downloads' + + # Specific build version + self.appversion = '' + self.appimagefilename = {} + self.zsyncfilename = {} + + # Creating a tempfile + self.builddir = tempfile.mkdtemp() + self.tarballs = {} + self.built = { u'x86': False, u'x86_64': False } + + # Preparing the default for the relative path on the storage for + # different versions. + # The path will evaluated as part of the check() function, as it is + # understood the storage_path can be changed before that phase. + self.relative_path = [] + self.full_path = '' + self.baseurl = '' + + + def calculate(self): + """Calculate exclusions and other variables.""" + # AppName + self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' + + # Calculating languagepart + self.languagepart = "." + if ',' in self.language: + self.languagepart += self.language.replace(',', '-') + else: + self.languagepart += self.language + + # Calculating help part + self.helppart = '.help' if self.offline_help else '' + + # Building the required names + for arch in Build.ARCHSTD: + self.appimagefilename[arch] = self.__gen_appimagefilename__(self.version, arch) + self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' + + # Mandate to the private function to calculate the full_path available + # for the storage and the checks. + self.__calculate_full_path__() + + + def __gen_appimagefilename__(self, version, arch): + """Generalize the construction of the name of the app.""" + self.appversion = version + self.languagepart + self.helppart + return self.appname + f'-{self.appversion}-{arch}.AppImage' + + + def __calculate_full_path__(self): + """Calculate relative path of the build, based on internal other variables.""" + if len(self.relative_path) == 0: + if self.query == 'daily': + self.relative_path.append('daily') + elif self.query == 'primageerelease': + self.relative_path.append('prerelease') + + # Not the same check, an additional one + if self.portable: + self.relative_path.append('portable') + + fullpath_arr = self.storage_path.split('/') + # Joining relative path only if it is not null + if len(self.relative_path) > 0: + fullpath_arr.extend(self.relative_path) + self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) + + + def check(self): + """Checking if the requested AppImage has been already built.""" + if not len(self.appimagefilename) == 2: + self.calculate() + + for arch in self.arch: + print(f"Searching for {self.appimagefilename[arch]}") + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + + if "No such file or directory" in res.stderr: + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout and len(res.stdout.strip("\n")) > 0: + # All good, the command was executed fine. + print(f"Build for {self.version} found.") + self.built[arch] = True + + if self.built[arch]: + print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + + + def download(self): + """Downloads the contents of the URL as it was a folder.""" + print(f"Started downloads for {self.version}. Please wait.") + for arch in self.arch: + # Checking if a valid path has been provided + if self.url[arch] == '-': + print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.") + # Faking already built it so to skip other checks. + self.built[arch] = True + continue + + if self.built[arch]: + print(f"A build for {arch} was already found. Skipping specific packages.") + continue + + # Identifying downloads + contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") + self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] + tarballs = self.tarballs[arch] + maintarball = tarballs[0] + + # Create and change directory to the download location + os.makedirs(self.download_path, exist_ok = True) + os.chdir(self.download_path) + for archive in tarballs: + # If the archive is already there, do not do anything. + if os.path.exists(archive): + continue + + # Download the archive + try: + urllib.request.urlretrieve(self.url[arch] + archive, archive) + except: + print(f"Failed to download {archive}.") + + print(f"Finished downloads for {self.version}.") + + def build(self): + """Building all the versions.""" + + for arch in self.arch: + if self.built[arch]: + # Already built for arch or path not available. User has already been warned. + continue + + # Preparation tasks + self.appnamedir = os.path.join(self.builddir, self.appname) + os.makedirs(self.appnamedir, exist_ok=True) + # And then cd to the appname folder. + os.chdir(self.appnamedir) + # Download appimagetool from github + appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage" + urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') + os.chmod('appimagetool', 0o755) + + # Build the requested version. + self.__unpackbuild__(arch) + + + def __unpackbuild__(self, arch): + # We start by filtering out tarballs from the list + buildtarballs = [ self.tarballs[arch][0] ] + + # Let's process standard languages and append results to the + # buildtarball + if self.language == 'basic': + if self.offline_help: + buildtarballs.extend([ x for x in self.tarballs[arch] if 'pack_en-GB' in x ]) + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) + elif self.language == 'standard': + for lang in Build.LANGSTD: + if self.offline_help: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack_' + lang) in x ]) + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack_' + lang) in x ]) + elif self.language == 'full': + if self.offline_help: + # We need also all help. Let's replace buildtarball with the + # whole bunch + buildtarballs = self.tarballs[arch] + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) + else: + # Looping for each language in self.language + for lang in self.language.split(","): + if self.offline_help: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + else: + buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) + + os.chdir(self.appnamedir) + + # Unpacking the tarballs + for archive in buildtarballs: + subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + + # create appimagedir + self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') + os.makedirs(self.appimagedir, exist_ok = True) + + # At this point, let's decompress the deb packages + subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) + + if self.portable: + subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) + + # Changing desktop file + subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) + + subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) + + # Find the name of the binary called in the desktop file. + binaryname = '' + with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: + a = d.readlines() + for line in a: + if re.match(r'^Exec', line): + binaryname = line.split('=')[-1].split(' ')[0] + # Esci al primo match + break + #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') + #binaryname = binary_exec.stdout.strip("\n") + + bindir=os.path.join(self.appimagedir, 'usr', 'bin') + os.makedirs(bindir, exist_ok = True) + subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) + + # Download AppRun from github + apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" + dest = os.path.join(self.appimagedir, 'AppRun') + urllib.request.urlretrieve(apprunurl, dest) + os.chmod(dest, 0o755) + + # Dealing with extra options + buildopts = [] + if self.sign: + buildopts.append('--sign') + + # adding zsync build if updatable + if self.updatable: + buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'") + + buildopts_str = str.join(' ', buildopts) + # Build the number-specific build + subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) + + print(f"Built AppImage version {self.appversion}") + + # Cleanup phase, before new run. + for deb in glob.glob(self.appnamedir + '/*.deb'): + os.remove(deb) + subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) + + + def checksums(self): + """Create checksums of the built versions.""" + # Skip checksum if initally the build was already found in the storage directory + if all(self.built.values()): + return + + os.chdir(self.appnamedir) + for arch in self.arch: + for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: + # For any built arch, find out if a file exist. + self.__create_checksum__(item) + + + def __create_checksum__(self, file): + """Internal function to create checksum file.""" + checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') + if checksum.stdout: + with open(f"{file}.md5", 'w') as c: + c.write(checksum.stdout) + + def publish(self): + """Moves built versions to definitive storage.""" + if all(self.built.values()): + # All files are already present in the full_path + return + + os.chdir(self.appnamedir) + # Forcing creation of subfolders, in case there is a new build + os.makedirs(self.full_path, exist_ok = True) + for file in glob.glob("*.AppImage*"): + subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + + + def generalize_and_link(self): + """Creates the needed generalized files if needed.""" + # If called with a pointed version, no generalize and link necessary. + if not self.branch_version: + return + appimagefilename = {} + zsyncfilename = {} + + # Creating versions for short version and query text + versions = [ self.short_version, self.branch_version ] + for arch in Build.ARCHSTD: + # If already built, do not do anything. + if self.built[arch]: + continue + + os.chdir(self.full_path) + # if the appimage for the reported arch is not found, skip to next + # arch + if not os.path.exists(self.appimagefilename[arch]): + continue + + # Doing it both for short_name and for branchname + for version in versions: + appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + zsyncfilename[arch] = appimagefilename[arch] + '.zsync' + + # Create the symlink + print(f"Creating {appimagefilename[arch]} and checksums.") + if os.path.exists(appimagefilename[arch]): + os.unlink(appimagefilename[arch]) + os.symlink(self.appimagefilename[arch], appimagefilename[arch]) + # Create the checksum for the AppImage + self.__create_checksum__(appimagefilename[arch]) + # Do not continue if no zsync file is provided. + if not self.updatable: + continue + + print(f"Creating zsync file for version {version}.") + if os.path.exists(zsyncfilename[arch]): + os.unlink(zsyncfilename[arch]) + shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) + # Editing the zsyncfile + subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) + self.__create_checksum__(zsyncfilename[arch]) + + + def __del__(self): + """Destructor""" + # Cleaning up build directory + shutil.rmtree(self.builddir) diff --git a/loaih/script.py b/loaih/script.py index 5e8499f..30d7bbf 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -24,10 +24,10 @@ def getversion(query, jsonout): for q in queries: if '.' in q: # Numbered version. It is safe to send it to Build. - b.append(loaih.versions.Build(q)) + b.append(loaih.RemoteBuild(q)) else: # Named version. For safety, we call a helper method for a collection - b.extend(loaih.versions.Base.collectedbuilds(q)) + b.extend(loaih.Base.collectedbuilds(q)) if len(b) > 0: if jsonout: diff --git a/loaih/versions.py b/loaih/versions.py deleted file mode 100644 index 3192438..0000000 --- a/loaih/versions.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import urllib.request -from lxml import etree -from packaging.version import parse as parse_version -import datetime - -class Definitions(object): - DOWNLOADPAGE = "https://www.libreoffice.org/download/download/" - ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" - RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" - DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" - PRERELEASE = "https://dev-builds.libreoffice.org/pre-releases/deb/x86_64/" - - SELECTORS = { - 'still': { - 'URL': DOWNLOADPAGE, - 'xpath': '(//span[@class="dl_version_number"])[last()]/text()' - }, - 'fresh': { - 'URL': DOWNLOADPAGE, - 'xpath': '(//span[@class="dl_version_number"])[1]/text()' - }, - 'prerelease': { - 'URL': DOWNLOADPAGE, - 'xpath': '//p[@class="lead_libre"][last()]/following-sibling::ul[last()]/li/a/text()' - }, - 'daily': { - 'URL': DAILY, - 'xpath': '//td/a' - } - } - -class Base(object): - # Class for static methods which might be useful even outside the build - # scripts. - - @staticmethod - def dailyurl(date = datetime.datetime.today()): - """Returns the URL for the latest valid daily build.""" - # As per other parts of the build, we need to maintain an URL also for - # x86 versions that it isn't really provided. - # As such, the return value must be a dictionary - - # Get the anchor for today's builds - a = etree.HTML(urllib.request.urlopen(Definitions.DAILY).read()).xpath("//td/a[contains(text(), '" + date.strftime('%Y-%m-%d') + "')]/text()") - if len(a) == 0: - # No results found, no version found, let's return a - return { 'x86': '-', 'x86_64': '-' } - - # On the contrary, more than a version is found. let's order the - # list and get the latest item - return { 'x86': '-', 'x86_64': Definitions.SELECTORS['daily']['URL'] + sorted(a)[-1] } - - @staticmethod - def dailyver(date = datetime.datetime.today()): - """Returns versions present on the latest daily build.""" - url = Base.dailyurl(date)['x86_64'] - # If no daily releases has been provided yet, return empty - if url == '-': - return [] - - # Rerun the page parsing, this time to find out the versions built - b = etree.HTML(urllib.request.urlopen(url).read()).xpath("//td/a[contains(text(), '_deb.tar.gz')]/text()") - # This should have returned the main package for a version, but can - # have returned multiple ones, so let's treat it as a list - return [ x.split('_')[1] for x in b ] - - @staticmethod - def namedver(query): - """Gets the version for a specific named version.""" - - if query == 'daily' or query == 'yesterday': - # Daily needs double parsing for the same result to apply. - # We first select today's build anchor: - date = datetime.datetime.today() - if query == 'yesterday': - # Use yesterdays' date for testing purposes. - date += datetime.timedelta(days=-1) - return Base.dailyver(date) - - # In case the query isn't for daily - return etree.HTML(urllib.request.urlopen(Definitions.SELECTORS[query]['URL']).read()).xpath(Definitions.SELECTORS[query]['xpath']) - - @staticmethod - def fullversion(version): - """Get latest full version from Archive based on partial version.""" - versionlist = etree.HTML(urllib.request.urlopen(Definitions.ARCHIVE).read()).xpath(f"//td/a[starts-with(text(), '{version}')]/text()") - cleanlist = sorted([ x.strip('/') for x in versionlist ]) - - # Sorting, then returning the last version - return cleanlist[-1] - - @staticmethod - def urlfromqueryandver(query, version): - """Returns the fetching URL based on the queried version and the numeric version of it.""" - # This has the purpose to simplify and explain how the releases are - # layed out. - - # If the query tells about daily or 'yesterday' (for testing purposes), - # we might ignore versions and return the value coming from dailyurl: - if query == 'daily': - return Base.dailyurl() - if query == 'yesterday': - date = datetime.datetime.today() + datetime.timedelta(days=-1) - return Base.dailyurl(date) - - # All other versions will be taken from Archive, as such we need a full - # version. - - # If the version has only 2 points in it (or splits into three parts by '.'), that's not a full version and we will call the getlatestver() function - fullversion = version - if len(version.split('.')) <= 3: - fullversion = Base.fullversion(version) - - # So the final URL is the Archive one, plus the full versions, plus a - # final '/deb/' - and an arch subfolder - baseurl = Definitions.ARCHIVE + fullversion + '/deb/' - retval = {} - - # x86 binaries are not anymore offered after 6.3.0. - if parse_version(version) < parse_version('6.3.0'): - retval['x86'] = baseurl + 'x86/' - else: - retval['x86'] = '-' - - retval['x86_64'] = baseurl + 'x86_64/' - - return retval - - @staticmethod - def collectedbuilds(query): - """Creates a list of Builds based on each namedver found.""" - retval = [] - a = Base.namedver(query) - if isinstance(a, list) and len(a) > 1: - retval = [ Build(query, version) for version in Base.namedver(query) ] - else: - retval.append(Build(query)) - - return retval - - -class Build(object): - - def __init__(self, query, version = None): - """Should simplify the single builded version.""" - self.query = query - self.version = '' - self.basedirurl = { 'x86': '-', 'x86_64': '-' } - - if not '.' in self.query: - # Named version. - # Let's check if a specific version was requested. - if version: - self.version = version - else: - # In case it was not requested, we will carry on the generic - # namedver() query. - # If the results are more than one, we'll take the latest (since we are requested to provide a single build). - a = Base.namedver(self.query) - - if isinstance(a, list) and len(a) == 0: - # No results from the query - let's return default values - return - - if len(a) == 1: - # version is a single one. - self.version = a[0] - else: - # In this case, we will select the latest release. - self.version = sorted(a)[-1] - else: - # In case of numbered queries, put it as initial version - self.version = self.query - - if len(self.version.split('.')) < 4: - # If not 4 dotted, let's search for the 4 dotted version - self.version = Base.fullversion(self.version) - - self.basedirurl = Base.urlfromqueryandver(self.query, self.version) - - def todict(self): - return { - 'query': self.query, - 'version': self.version, - 'basedirurl': self.basedirurl - } - - def __str__(self): - return f"""query: {self.query} -version: {self.version} -x86: {self.basedirurl['x86']} -x86_64: {self.basedirurl['x86_64']}""" From d0f73d0f1618b959e846a4f3ecc6f2cccf56f1b2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 23:31:06 +0200 Subject: [PATCH 127/176] Gestite multiple versioni. Ora anche prerelease e daily sono supportati. --- loaih/__init__.py | 52 ++++++++++++++++------------- loaih/build.py | 24 +++++++------- loaih/script.py | 84 ++++++++++++++++++++++------------------------- 3 files changed, 81 insertions(+), 79 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index b102568..d15ccc8 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -110,9 +110,9 @@ class Base(object): # version. # If the version has only 2 points in it (or splits into three parts by '.'), that's not a full version and we will call the getlatestver() function - fullversion = version - if len(version.split('.')) <= 3: - fullversion = Base.fullversion(version) + fullversion = str(version) + if len(fullversion.split('.')) <= 3: + fullversion = str(Base.fullversion(version)) # So the final URL is the Archive one, plus the full versions, plus a # final '/deb/' - and an arch subfolder @@ -120,7 +120,7 @@ class Base(object): retval = {} # x86 binaries are not anymore offered after 6.3.0. - if parse_version(version) < parse_version('6.3.0'): + if parse_version(fullversion) < parse_version('6.3.0'): retval['x86'] = baseurl + 'x86/' else: retval['x86'] = '-' @@ -131,15 +131,20 @@ class Base(object): @staticmethod def collectedbuilds(query): - """Creates a list of Builds based on each namedver found.""" + """Creates a list of Builds based on each queried version found.""" retval = [] - a = Base.namedver(query) - if isinstance(a, list) and len(a) > 1: - retval = [ Build(query, version) for version in Base.namedver(query) ] - else: + if '.' in query: + # Called with a numeric query. Pass it to RemoteBuild retval.append(RemoteBuild(query)) + else: + # Named query + a = Base.namedver(query) + if isinstance(a, list) and len(a) > 1: + retval.extend([ RemoteBuild(query, version) for version in a ]) + else: + retval.append(RemoteBuild(query)) - return retval + return sorted(retval, key=lambda x: x.version) class RemoteBuild(object): @@ -150,32 +155,33 @@ class RemoteBuild(object): self.version = '' self.basedirurl = { 'x86': '-', 'x86_64': '-' } + if version and isinstance(version, str): + self.version = version + if not '.' in self.query: # Named version. # Let's check if a specific version was requested. - if version: - self.version = version - else: + if self.version == '': # In case it was not requested, we will carry on the generic # namedver() query. # If the results are more than one, we'll take the latest (since we are requested to provide a single build). a = Base.namedver(self.query) - if isinstance(a, list) and len(a) == 0: - # No results from the query - let's return default values - return + if isinstance(a, list): + if len(a) == 1: + # version is a single one. + self.version = a[0] + else: + # In this case, we will select the latest release. + self.version = sorted(a)[-1] - if len(a) == 1: - # version is a single one. - self.version = a[0] - else: - # In this case, we will select the latest release. - self.version = sorted(a)[-1] + # If the version has already a version, as requested by user, + # continue using that version else: # In case of numbered queries, put it as initial version self.version = self.query - if len(self.version.split('.')) < 4: + if len(str(self.version).split('.')) < 4: # If not 4 dotted, let's search for the 4 dotted version self.version = Base.fullversion(self.version) diff --git a/loaih/build.py b/loaih/build.py index 59e648a..fec92d3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -5,25 +5,26 @@ import loaih from lxml import etree import tempfile, os, sys, glob, subprocess, shutil, re, shlex -class Build(object): +class Collection(list): + + def __init__(self, query, arch = ['x86', 'x86_64']): + """Build a list of version to check/build for this round.""" + super().__init__() + self.extend([ Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) ]) + +class Build(loaih.RemoteBuild): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGBASIC = [ 'en-GB' ] ARCHSTD = [ u'x86', u'x86_64' ] - def __init__(self, query, arch): - """Build all versions that can be found in the indicated repo.""" - self.query = query + def __init__(self, query, arch, version = None): + super().__init__(query, version) self.arch = arch - - # Getting versions and so on - v = versions.BuildVersion(self.query) - self.version = v.version - print(f"Debug {self.version}") self.short_version = str.join('.', self.version.split('.')[0:2]) self.branch_version = None if not '.' in self.query: self.branch_version = self.query - self.url = v.basedirurl + self.url = self.basedirurl # Other default values self.language = 'basic' @@ -52,7 +53,6 @@ class Build(object): self.full_path = '' self.baseurl = '' - def calculate(self): """Calculate exclusions and other variables.""" # AppName @@ -89,7 +89,7 @@ class Build(object): if len(self.relative_path) == 0: if self.query == 'daily': self.relative_path.append('daily') - elif self.query == 'primageerelease': + elif self.query == 'prerelease': self.relative_path.append('prerelease') # Not the same check, an additional one diff --git a/loaih/script.py b/loaih/script.py index 30d7bbf..40ddf36 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -3,7 +3,7 @@ import click import yaml -import loaih +import loaih, loaih.build import re, sys, json @click.group() @@ -22,12 +22,7 @@ def getversion(query, jsonout): queries.append(query) for q in queries: - if '.' in q: - # Numbered version. It is safe to send it to Build. - b.append(loaih.RemoteBuild(q)) - else: - # Named version. For safety, we call a helper method for a collection - b.extend(loaih.Base.collectedbuilds(q)) + b.extend(loaih.Base.collectedbuilds(q)) if len(b) > 0: if jsonout: @@ -66,22 +61,50 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path # generic default. for build in config['builds']: # Loop a run for each build. - obj = loaih.Build(build['query'], arches) + collection = loaih.build.Collection(build['query'], arches) + for obj in collection: + # Configuration phase + obj.language = build['language'] + obj.offline_help = build['offline_help'] + obj.portable = build['portable'] + obj.updatable = True + obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' + obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + + if 'sign' in config['data'] and config['data']['sign']: + obj.sign = True + + # Build phase + obj.calculate() + if not 'force' in config['data'] or not config['data']['force']: + obj.check() + + obj.download() + obj.build() + obj.checksums() + obj.publish() + obj.generalize_and_link() + del obj + + else: + collection = loaih.build.Collection(query, arches) + for obj in collection: # Configuration phase - obj.language = build['language'] - obj.offline_help = build['offline_help'] - obj.portable = build['portable'] - obj.updatable = True - obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' - obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + obj.language = language + obj.offline_help = offline + obj.portable = portable + obj.updatable = updatable + obj.storage_path = repo_path + obj.download_path = download_path - if 'sign' in config['data'] and config['data']['sign']: + if sign: obj.sign = True - # Build phase + # Running phase obj.calculate() - if not 'force' in config['data'] or not config['data']['force']: + + if check: obj.check() obj.download() @@ -90,30 +113,3 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.publish() obj.generalize_and_link() del obj - - else: - obj = loaih.Build(query, arches) - - # Configuration phase - obj.language = language - obj.offline_help = offline - obj.portable = portable - obj.updatable = updatable - obj.storage_path = repo_path - obj.download_path = download_path - - if sign: - obj.sign = True - - # Running phase - obj.calculate() - - if check: - obj.check() - - obj.download() - obj.build() - obj.checksums() - obj.publish() - obj.generalize_and_link() - del obj From 561290fc63afc2fa4b00e5f140ef368810e05e96 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 23:32:26 +0200 Subject: [PATCH 128/176] Aggiunta buildfile per prerelease. --- prerelease.yml | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 prerelease.yml diff --git a/prerelease.yml b/prerelease.yml new file mode 100644 index 0000000..402f385 --- /dev/null +++ b/prerelease.yml @@ -0,0 +1,67 @@ +--- +data: + repo: /mnt/appimage + download: /var/tmp/downloads + force: no + sign: yes + +builds: + - query: prerelease + language: basic + offline_help: no + portable: no + + - query: prerelease + language: basic + offline_help: yes + portable: no + + - query: prerelease + language: basic + offline_help: no + portable: yes + + - query: prerelease + language: basic + offline_help: yes + portable: yes + + - query: prerelease + language: standard + offline_help: no + portable: no + + - query: prerelease + language: standard + offline_help: yes + portable: no + + - query: prerelease + language: standard + offline_help: no + portable: yes + + - query: prerelease + language: standard + offline_help: yes + portable: yes + + - query: prerelease + language: full + offline_help: no + portable: no + + - query: prerelease + language: full + offline_help: yes + portable: no + + - query: prerelease + language: full + offline_help: no + portable: yes + + - query: prerelease + language: full + offline_help: yes + portable: yes From 102dfc192ddf152760b1d5adf0943f64c68e6359 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 23:34:11 +0200 Subject: [PATCH 129/176] Aumentata versione release in setup.py. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4ad69f..c41059f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup,find_packages setup( name="loaih", - version="1.1.0", + version="1.2.0", description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", author="Emiliano Vavassori", author_email="syntaxerrormmm@libreoffice.org", From fb92183b4bf70eebd12f22e9212903dafd3604bf Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 30 Apr 2022 23:35:41 +0200 Subject: [PATCH 130/176] Ignoro le cartelle pycache. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ff7d489..eaa4aae 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ venv build dist loaih.egg-info +**/__pycache__ From eafcc9bc4ab7b74b77a4ff1ff66a92c802ed3355 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 1 May 2022 13:29:04 +0200 Subject: [PATCH 131/176] Ignorate le generalizzazioni per prerelease e daily. Aggiunto buildfile per daily. --- daily.yml | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ loaih/build.py | 5 ++++ 2 files changed, 72 insertions(+) create mode 100644 daily.yml diff --git a/daily.yml b/daily.yml new file mode 100644 index 0000000..2f2f521 --- /dev/null +++ b/daily.yml @@ -0,0 +1,67 @@ +--- +data: + repo: /mnt/appimage + download: /var/tmp/downloads + force: no + sign: yes + +builds: + - query: daily + language: basic + offline_help: no + portable: no + + - query: daily + language: basic + offline_help: yes + portable: no + + - query: daily + language: basic + offline_help: no + portable: yes + + - query: daily + language: basic + offline_help: yes + portable: yes + + - query: daily + language: standard + offline_help: no + portable: no + + - query: daily + language: standard + offline_help: yes + portable: no + + - query: daily + language: standard + offline_help: no + portable: yes + + - query: daily + language: standard + offline_help: yes + portable: yes + + - query: daily + language: full + offline_help: no + portable: no + + - query: daily + language: full + offline_help: yes + portable: no + + - query: daily + language: full + offline_help: no + portable: yes + + - query: daily + language: full + offline_help: yes + portable: yes diff --git a/loaih/build.py b/loaih/build.py index fec92d3..7f1513a 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -320,6 +320,11 @@ class Build(loaih.RemoteBuild): # If called with a pointed version, no generalize and link necessary. if not self.branch_version: return + + # If a prerelease or a daily version, either. + if self.query == 'daily' or self.query == 'prerelease' + return + appimagefilename = {} zsyncfilename = {} From 3656b7d0a6e1d584fc0081f0b15c1f3264864b7b Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sun, 1 May 2022 13:30:26 +0200 Subject: [PATCH 132/176] Correzione di sintassi. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 7f1513a..bbd0631 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -322,7 +322,7 @@ class Build(loaih.RemoteBuild): return # If a prerelease or a daily version, either. - if self.query == 'daily' or self.query == 'prerelease' + if self.query == 'daily' or self.query == 'prerelease': return appimagefilename = {} From db4c957e1d4586f6cdef829e9d2987b6cd6b9560 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 12 May 2022 22:57:11 +0200 Subject: [PATCH 133/176] Fix per numero di build vuoto. --- loaih/__init__.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index d15ccc8..f424f60 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -87,10 +87,13 @@ class Base(object): def fullversion(version): """Get latest full version from Archive based on partial version.""" versionlist = etree.HTML(urllib.request.urlopen(Definitions.ARCHIVE).read()).xpath(f"//td/a[starts-with(text(), '{version}')]/text()") - cleanlist = sorted([ x.strip('/') for x in versionlist ]) + if versionlist: + cleanlist = sorted([ x.strip('/') for x in versionlist ]) - # Sorting, then returning the last version - return cleanlist[-1] + # Sorting, then returning the last version + return cleanlist[-1] + + return None @staticmethod def urlfromqueryandver(query, version): @@ -139,6 +142,11 @@ class Base(object): else: # Named query a = Base.namedver(query) + + if not a: + # a is empty + return retval + if isinstance(a, list) and len(a) > 1: retval.extend([ RemoteBuild(query, version) for version in a ]) else: @@ -168,6 +176,10 @@ class RemoteBuild(object): a = Base.namedver(self.query) if isinstance(a, list): + # if the number of versions is zero, return and exit + if not a: + return None + if len(a) == 1: # version is a single one. self.version = a[0] From 100fa2e2acdb647bb57e03ec89d9a62fe7a4154d Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 4 Jan 2023 23:06:48 +0100 Subject: [PATCH 134/176] Provo pipeline con drone. --- .drone.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..d78b967 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +name: default + +steps: + - name: compile + image: python + commands: + - pip install wheel + - python setup.py bdist_wheel + - mkdir out + - cp dist/*.whl out/ From a84641c6ea68312e4c77ae7b5c0900413985041f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 4 Jan 2023 23:11:27 +0100 Subject: [PATCH 135/176] Ancora modifiche alla pipeline. --- .drone.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.drone.yml b/.drone.yml index d78b967..bab7760 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,3 +10,14 @@ steps: - python setup.py bdist_wheel - mkdir out - cp dist/*.whl out/ + + - name: release + image: plugins/gitea-release + settings: + api_key: + from_secret: gitea-deploy + base_url: https://git.sys42.eu + files: out/*.whl + checksum: + - md5 + draft: true From 51079c81d4b106d1c596dd528da8303b780245d0 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 4 Jan 2023 23:12:24 +0100 Subject: [PATCH 136/176] Modifiche alla pipeline. --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index bab7760..839dcae 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,6 +10,8 @@ steps: - python setup.py bdist_wheel - mkdir out - cp dist/*.whl out/ + when: + event: tag - name: release image: plugins/gitea-release @@ -21,3 +23,5 @@ steps: checksum: - md5 draft: true + when: + event: tag From df70a1f954cac5e8af7f38dbfde0f98b7b3f56c7 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 4 Jan 2023 23:15:07 +0100 Subject: [PATCH 137/176] Ancora modifiche alla pipeline. --- .drone.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index 839dcae..47336c6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,10 +18,10 @@ steps: settings: api_key: from_secret: gitea-deploy - base_url: https://git.sys42.eu - files: out/*.whl - checksum: - - md5 - draft: true + base_url: https://git.sys42.eu/ + files: out/*.whl + checksum: + - md5 + draft: true when: event: tag From 7d9a701caeafd1e51a01cbd948d94f5ba62bd1f1 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 4 Jan 2023 23:37:24 +0100 Subject: [PATCH 138/176] Aggiunte alla pipeline. --- .drone.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.drone.yml b/.drone.yml index 47336c6..1963b47 100644 --- a/.drone.yml +++ b/.drone.yml @@ -25,3 +25,16 @@ steps: draft: true when: event: tag + + - name: handycopy + image: drillster/drone-rsync + settings: + hosts: deimos.sys42.eu + user: syntaxerrormmm + port: 45454 + key: + from_secret: fisso-ssh-key + source: out/*.whl + target: ~/ + when: + event: tag From 2c19eefa056d0170e0ce18711c495d6f9c0d597a Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Wed, 4 Jan 2023 23:44:10 +0100 Subject: [PATCH 139/176] Rimozione passaggi superflui pipeline. --- .drone.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.drone.yml b/.drone.yml index 1963b47..fcc1e05 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,8 +8,6 @@ steps: commands: - pip install wheel - python setup.py bdist_wheel - - mkdir out - - cp dist/*.whl out/ when: event: tag @@ -19,22 +17,8 @@ steps: api_key: from_secret: gitea-deploy base_url: https://git.sys42.eu/ - files: out/*.whl - checksum: - - md5 + files: dist/*.whl + checksum: md5 draft: true when: event: tag - - - name: handycopy - image: drillster/drone-rsync - settings: - hosts: deimos.sys42.eu - user: syntaxerrormmm - port: 45454 - key: - from_secret: fisso-ssh-key - source: out/*.whl - target: ~/ - when: - event: tag From df5012eedd1ca3ac79ae2c35f95125c535880af6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:07:12 +0100 Subject: [PATCH 140/176] Prima implementazione controllo build remota e caricamento con rsync + ssh. --- loaih/build.py | 80 +++++++++++++++++++++++++++++++++++++------------ loaih/script.py | 9 +++++- 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index bbd0631..a7a2a04 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -32,6 +32,9 @@ class Build(loaih.RemoteBuild): self.portable = False self.updatable = True self.sign = True + self.remoterepo = False + self.remote_host = '' + self.remote_path = '' self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' @@ -90,7 +93,7 @@ class Build(loaih.RemoteBuild): if self.query == 'daily': self.relative_path.append('daily') elif self.query == 'prerelease': - self.relative_path.append('prerelease') + self.relative_path.append('prerelease') # Not the same check, an additional one if self.portable: @@ -105,25 +108,44 @@ class Build(loaih.RemoteBuild): def check(self): """Checking if the requested AppImage has been already built.""" - if not len(self.appimagefilename) == 2: + if len(self.appimagefilename) != 2: self.calculate() for arch in self.arch: - print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') - if "No such file or directory" in res.stderr: - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue + # First, check if by metadata the repo is remote or not. + if self.remoterepo or 'http' in self.storage_path: + self.remoterepo = True + # Remote storage. I have to query a remote site to know if it + # was already built. + name = self.appimagefilename[arch] + matching = etree.HTML(urllib.request.urlopen(str.join('/', + self.relative_path.insert(0, self.storage_path) + )).read()).xpath( + f"//a[contains(@href, '{name}')/@href" + ) - if res.stdout and len(res.stdout.strip("\n")) > 0: - # All good, the command was executed fine. - print(f"Build for {self.version} found.") - self.built[arch] = True + if len(matching) > 0: + # Already built. + self.built[arch] = True + + else: + # Repo is local + print(f"Searching for {self.appimagefilename[arch]}") + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + + if "No such file or directory" in res.stderr: + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout and len(res.stdout.strip("\n")) > 0: + # All good, the command was executed fine. + print(f"Build for {self.version} found.") + self.built[arch] = True if self.built[arch]: - print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") def download(self): @@ -309,13 +331,33 @@ class Build(loaih.RemoteBuild): return os.chdir(self.appnamedir) - # Forcing creation of subfolders, in case there is a new build - os.makedirs(self.full_path, exist_ok = True) - for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + + # Two cases here: local and remote storage_path. + if self.remoterepo: + # Remote first. + # Build destination directory + if len(self.relative_path) > 0: + remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) + else: + remotepath = str.join('/', [ self.remote_path, '' ]) + try: + subprocess.run( + shlex.split( + f"rsync -avz -e ssh *.AppImage* {self.remote_host}:{remotepath}" + ) + ) + finally: + pass + + else: + # Local + # Forcing creation of subfolders, in case there is a new build + os.makedirs(self.full_path, exist_ok = True) + for file in glob.glob("*.AppImage*"): + subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - def generalize_and_link(self): + def generalize_and_link(self, chdir = self.full_path): """Creates the needed generalized files if needed.""" # If called with a pointed version, no generalize and link necessary. if not self.branch_version: @@ -335,7 +377,7 @@ class Build(loaih.RemoteBuild): if self.built[arch]: continue - os.chdir(self.full_path) + os.chdir(chdir) # if the appimage for the reported arch is not found, skip to next # arch if not os.path.exists(self.appimagefilename[arch]): diff --git a/loaih/script.py b/loaih/script.py index 40ddf36..46afbe0 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -71,6 +71,10 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.updatable = True obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + if 'http' in obj.storage_path: + obj.remoterepo = True + obj.remote_host = config['data']['remote_host'] if 'remote_host' in config['data'] and config['data']['remote_host'] else 'ciccio.libreitalia.org' + obj.remote_path = config['data']['remote_path'] if 'remote_path' in config['data'] and config['data']['remote_path'] else '/var/lib/nethserver/vhost/appimages' if 'sign' in config['data'] and config['data']['sign']: obj.sign = True @@ -83,8 +87,11 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.download() obj.build() obj.checksums() + if obj.remoterepo: + obj.generalize_and_link(obj.appnamedir) obj.publish() - obj.generalize_and_link() + if not obj.remoterepo: + obj.generalize_and_link() del obj else: From cbaf6b2e3cee67ab850c28be43ef574a8659b4c2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:08:35 +0100 Subject: [PATCH 141/176] Corretto anche il file di build per il test. --- test.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test.yml b/test.yml index faf4464..768cdd6 100644 --- a/test.yml +++ b/test.yml @@ -1,12 +1,14 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: yes - sign: yes + force: true + sign: true builds: - query: fresh language: basic - offline_help: no - portable: no + offline_help: false + portable: false From 9cf311948922b8899d14ca821e3a14fb9470ca90 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:14:28 +0100 Subject: [PATCH 142/176] Correzione sintassi in build. --- loaih/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index a7a2a04..279c3c6 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -357,7 +357,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - def generalize_and_link(self, chdir = self.full_path): + def generalize_and_link(self, chdir = 'default'): """Creates the needed generalized files if needed.""" # If called with a pointed version, no generalize and link necessary. if not self.branch_version: @@ -367,6 +367,9 @@ class Build(loaih.RemoteBuild): if self.query == 'daily' or self.query == 'prerelease': return + if chdir == 'default': + chdir = self.full_path + appimagefilename = {} zsyncfilename = {} From ae9668554a68606a0c77293855c8fd5a073926f6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:24:37 +0100 Subject: [PATCH 143/176] Cambiamenti per esecuzione corretta verifica online. --- loaih/build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 279c3c6..8124c7a 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -119,9 +119,13 @@ class Build(loaih.RemoteBuild): # Remote storage. I have to query a remote site to know if it # was already built. name = self.appimagefilename[arch] - matching = etree.HTML(urllib.request.urlopen(str.join('/', - self.relative_path.insert(0, self.storage_path) - )).read()).xpath( + if len(self.relative_path) > 0: + path_arr = self.relative_path.insert(0, self.storage_path) + else: + path_arr = [ self.storage_path, '' ] + matching = etree.HTML(urllib.request.urlopen( + str.join('/', path_arr) + ).read()).xpath( f"//a[contains(@href, '{name}')/@href" ) From da31e1655bc6cbe87c29dbe26abd60d2d6bc0a53 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:30:06 +0100 Subject: [PATCH 144/176] Aggiunta codice debug. --- loaih/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 8124c7a..6069c14 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -123,8 +123,10 @@ class Build(loaih.RemoteBuild): path_arr = self.relative_path.insert(0, self.storage_path) else: path_arr = [ self.storage_path, '' ] + path = str.join('/', path_arr) + print(f"DEBUG - Name: {name}, URL: {path}") matching = etree.HTML(urllib.request.urlopen( - str.join('/', path_arr) + str.join('/', path) ).read()).xpath( f"//a[contains(@href, '{name}')/@href" ) From 38a78860b04d716af66654d60f5014ef5067fe06 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:33:26 +0100 Subject: [PATCH 145/176] Fix procedura dopo codice debug. --- loaih/build.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 6069c14..39ba440 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -125,9 +125,7 @@ class Build(loaih.RemoteBuild): path_arr = [ self.storage_path, '' ] path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") - matching = etree.HTML(urllib.request.urlopen( - str.join('/', path) - ).read()).xpath( + matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( f"//a[contains(@href, '{name}')/@href" ) From 05533bf5e2511cba9175c370f4098dc9240e4336 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:36:53 +0100 Subject: [PATCH 146/176] Fix sintassi xpath. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 39ba440..b42acd3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -126,7 +126,7 @@ class Build(loaih.RemoteBuild): path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( - f"//a[contains(@href, '{name}')/@href" + f"//a[contains(@href, '{name}')]/@href" ) if len(matching) > 0: From 07a895c86c3543de036ee233b703dd2b65d27412 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:56:33 +0100 Subject: [PATCH 147/176] Fix checksum che torna valore non valido. --- loaih/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index b42acd3..8fb99a9 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -311,7 +311,7 @@ class Build(loaih.RemoteBuild): def checksums(self): """Create checksums of the built versions.""" # Skip checksum if initally the build was already found in the storage directory - if all(self.built.values()): + if all(self.built[arch] for arch in self.arch): return os.chdir(self.appnamedir) @@ -330,7 +330,7 @@ class Build(loaih.RemoteBuild): def publish(self): """Moves built versions to definitive storage.""" - if all(self.built.values()): + if all(self.built[arch] for arch in self.arch): # All files are already present in the full_path return From 9d259c4aa5cf5a9b8889361aa1ccee5e26891ad2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:59:55 +0100 Subject: [PATCH 148/176] Cambiata la logica di build in caso di assenza di build. --- loaih/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/script.py b/loaih/script.py index 46afbe0..bb1d7fe 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -87,7 +87,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.download() obj.build() obj.checksums() - if obj.remoterepo: + if obj.remoterepo and obj.appnamedir: obj.generalize_and_link(obj.appnamedir) obj.publish() if not obj.remoterepo: From f95c4d4b1d1312df674919731f9e5a04a9bbac38 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 18:28:11 +0100 Subject: [PATCH 149/176] Sistemazione upload con rsync. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 8fb99a9..be90f64 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -347,7 +347,7 @@ class Build(loaih.RemoteBuild): try: subprocess.run( shlex.split( - f"rsync -avz -e ssh *.AppImage* {self.remote_host}:{remotepath}" + f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}" ) ) finally: From a0c6217d95b3ba4d8869c7e65969057303318c29 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 19:58:45 +0100 Subject: [PATCH 150/176] Sistemazione build. --- loaih/build.py | 167 +++++++++++++++++++++++++++++++++++-------------- test.yml | 2 +- 2 files changed, 120 insertions(+), 49 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index be90f64..071127d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -1,21 +1,35 @@ #!/usr/bin/env python3 +# encoding: utf-8 +"""Classes and functions to build an AppImage.""" +import os +import glob +import subprocess +import shutil +import re +import shlex +import tempfile import urllib.request -import loaih from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re, shlex +import loaih class Collection(list): + """Aggregates metadata on a collection of builds.""" def __init__(self, query, arch = ['x86', 'x86_64']): """Build a list of version to check/build for this round.""" super().__init__() - self.extend([ Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) ]) + self.extend([ + Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) + ]) class Build(loaih.RemoteBuild): - LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] + """Builds a single version.""" + + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', + 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGBASIC = [ 'en-GB' ] - ARCHSTD = [ u'x86', u'x86_64' ] + ARCHSTD = [ 'x86', 'x86_64' ] def __init__(self, query, arch, version = None): super().__init__(query, version) @@ -37,17 +51,24 @@ class Build(loaih.RemoteBuild): self.remote_path = '' self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' + self.appnamedir = '' # Specific build version + self.appname = 'LibreOffice' self.appversion = '' + self.appimagedir = '' self.appimagefilename = {} self.zsyncfilename = {} + # Other variables by build + self.languagepart = '.' + self.language + self.helppart = '' + # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} - self.built = { u'x86': False, u'x86_64': False } - + self.built = { 'x86': False, 'x86_64': False } + # Preparing the default for the relative path on the storage for # different versions. # The path will evaluated as part of the check() function, as it is @@ -59,7 +80,8 @@ class Build(loaih.RemoteBuild): def calculate(self): """Calculate exclusions and other variables.""" # AppName - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' + if self.query in { 'prerelease', 'daily' }: + self.appname = 'LibreOfficeDev' # Calculating languagepart self.languagepart = "." @@ -69,7 +91,8 @@ class Build(loaih.RemoteBuild): self.languagepart += self.language # Calculating help part - self.helppart = '.help' if self.offline_help else '' + if self.offline_help: + self.helppart = '.help' # Building the required names for arch in Build.ARCHSTD: @@ -125,9 +148,11 @@ class Build(loaih.RemoteBuild): path_arr = [ self.storage_path, '' ] path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") - matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( - f"//a[contains(@href, '{name}')]/@href" - ) + matching = [] + with urllib.request.urlopen(path) as url: + matching = etree.HTML(url.read()).xpath( + f"//a[contains(@href, '{name}')]/@href" + ) if len(matching) > 0: # Already built. @@ -136,7 +161,11 @@ class Build(loaih.RemoteBuild): else: # Repo is local print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + command = f"find {self.full_path} -name {self.appimagefilename[arch]}" + res = subprocess.run(shlex.split(command), + capture_output=True, + env={ "LC_ALL": "C" }, + text=True, encoding='utf-8', check=True) if "No such file or directory" in res.stderr: # Folder is not existent: so the version was not built @@ -168,10 +197,15 @@ class Build(loaih.RemoteBuild): continue # Identifying downloads - contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] + contents = [] + with urllib.request.urlopen(self.url[arch]) as url: + contents = etree.HTML(url.read()).xpath("//td/a") + + self.tarballs[arch] = [ x.text + for x in contents + if x.text.endswith('tar.gz') and 'deb' in x.text + ] tarballs = self.tarballs[arch] - maintarball = tarballs[0] # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) @@ -184,8 +218,8 @@ class Build(loaih.RemoteBuild): # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) - except: - print(f"Failed to download {archive}.") + except Exception as error: + print(f"Failed to download {archive}: {error}.") print(f"Finished downloads for {self.version}.") @@ -213,7 +247,7 @@ class Build(loaih.RemoteBuild): def __unpackbuild__(self, arch): # We start by filtering out tarballs from the list - buildtarballs = [ self.tarballs[arch][0] ] + buildtarballs = [ self.tarballs[arch][0] ] # Let's process standard languages and append results to the # buildtarball @@ -239,50 +273,75 @@ class Build(loaih.RemoteBuild): # Looping for each language in self.language for lang in self.language.split(","): if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] + if 'pack' + lang in x ]) else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - + buildtarballs.extend([ x for x in self.tarballs[arch] + if 'langpack' + lang in x ]) + os.chdir(self.appnamedir) # Unpacking the tarballs for archive in buildtarballs: - subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + subprocess.run(shlex.split( + f"tar xzf {self.download_path}/{archive}"), check=True) # create appimagedir self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages - subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find .. -iname '*.deb' -exec dpkg -x {} . \;" + ), cwd=self.appimagedir, check=True) if self.portable: - subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find . -type f -iname 'bootstraprc' " + + r"-exec sed -i 's|^UserInstallation=.*|" + + r"UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version + ), cwd=self.appimagedir, check=True) # Changing desktop file - subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) - subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find . -iname startcenter.desktop -exec cp {} . \;" + ), cwd=self.appimagedir, check=True) - subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split( + f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " + + r"startcenter.desktop > startcenter.desktop" + ), cwd=self.appimagedir, check=True) + + subprocess.run(shlex.split( + r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " + + r"-exec cp {} . \;" + ), cwd=self.appimagedir, check=True) # Find the name of the binary called in the desktop file. binaryname = '' - with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: - a = d.readlines() - for line in a: + with open( + os.path.join(self.appimagedir, 'startcenter.desktop'), + 'r', encoding="utf-8" + ) as desktopfile: + for line in desktopfile.readlines(): if re.match(r'^Exec', line): binaryname = line.split('=')[-1].split(' ')[0] # Esci al primo match break + #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') #binaryname = binary_exec.stdout.strip("\n") bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) + subprocess.run(shlex.split( + r"find ../../opt -iname soffice -path '*program*' " + + r"-exec ln -sf {} ./%s \;" % binaryname + ), cwd=bindir, check=True) # Download AppRun from github - apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" + apprunurl = r"https://github.com/AppImage/AppImageKit/releases/" + apprunurl += f"download/continuous/AppRun-{arch}" dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) @@ -298,14 +357,19 @@ class Build(loaih.RemoteBuild): buildopts_str = str.join(' ', buildopts) # Build the number-specific build - subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) - + subprocess.run(shlex.split( + f"{self.appnamedir}/appimagetool {buildopts_str} -v " + + f"./{self.appname}.AppDir/" + ), env={ "VERSION": self.appversion }, check=True) + print(f"Built AppImage version {self.appversion}") # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) + subprocess.run(shlex.split( + r"find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+" + ), check=True) def checksums(self): @@ -323,10 +387,11 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') + checksum = subprocess.run(shlex.split(f"md5sum {file}"), + capture_output=True, text=True, encoding='utf-8', check=True) if checksum.stdout: - with open(f"{file}.md5", 'w') as c: - c.write(checksum.stdout) + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" @@ -345,11 +410,10 @@ class Build(loaih.RemoteBuild): else: remotepath = str.join('/', [ self.remote_path, '' ]) try: - subprocess.run( - shlex.split( - f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}" - ) - ) + subprocess.run(shlex.split( + r"rsync -rlIvz --munge-links *.AppImage* " + + f"{self.remote_host}:{remotepath}" + ), check=True) finally: pass @@ -358,7 +422,9 @@ class Build(loaih.RemoteBuild): # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + subprocess.run(shlex.split( + f"cp -f {file} {self.full_path}" + ), check=True) def generalize_and_link(self, chdir = 'default'): @@ -368,7 +434,7 @@ class Build(loaih.RemoteBuild): return # If a prerelease or a daily version, either. - if self.query == 'daily' or self.query == 'prerelease': + if self.query in { 'daily', 'prerelease' }: return if chdir == 'default': @@ -392,7 +458,9 @@ class Build(loaih.RemoteBuild): # Doing it both for short_name and for branchname for version in versions: - appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + appimagefilename[arch] = self.appname + '-' + version + appimagefilename[arch] += self.languagepart + self.helppart + appimagefilename[arch] += f'-{arch}.AppImage' zsyncfilename[arch] = appimagefilename[arch] + '.zsync' # Create the symlink @@ -411,7 +479,10 @@ class Build(loaih.RemoteBuild): os.unlink(zsyncfilename[arch]) shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile - subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) + subprocess.run(shlex.split( + r"sed --in-place 's/^Filename:.*$/Filename: " + + f"{appimagefilename[arch]}/' {zsyncfilename[arch]}" + ), check=True) self.__create_checksum__(zsyncfilename[arch]) diff --git a/test.yml b/test.yml index 768cdd6..8fcf1d7 100644 --- a/test.yml +++ b/test.yml @@ -8,7 +8,7 @@ data: sign: true builds: - - query: fresh + - query: 7.2.3 language: basic offline_help: false portable: false From 60b246c548409d61d08aa3c120d881e8f27b58ee Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:23:30 +0100 Subject: [PATCH 151/176] Correzione comando sbagliato da console. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 071127d..97c3c4c 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -308,7 +308,7 @@ class Build(loaih.RemoteBuild): ), cwd=self.appimagedir, check=True) subprocess.run(shlex.split( - f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " + + f"sed --in-place \'s:^Name=.*$:Name={self.appname}:\' " + r"startcenter.desktop > startcenter.desktop" ), cwd=self.appimagedir, check=True) From a316b85afead9ca5b2a5452ff93443814dd80f84 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:29:27 +0100 Subject: [PATCH 152/176] Correzione comando sbagliato da console - 1. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 97c3c4c..1b64535 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -310,7 +310,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split( f"sed --in-place \'s:^Name=.*$:Name={self.appname}:\' " + r"startcenter.desktop > startcenter.desktop" - ), cwd=self.appimagedir, check=True) + ), cwd=self.appimagedir, check=False) subprocess.run(shlex.split( r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " + From a0c4fbcad097f3f32b619129db4813c75350ecb9 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:39:29 +0100 Subject: [PATCH 153/176] Correzione rsync. --- loaih/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 1b64535..0a5b820 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -309,7 +309,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split( f"sed --in-place \'s:^Name=.*$:Name={self.appname}:\' " + - r"startcenter.desktop > startcenter.desktop" + r"startcenter.desktop" ), cwd=self.appimagedir, check=False) subprocess.run(shlex.split( @@ -413,7 +413,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split( r"rsync -rlIvz --munge-links *.AppImage* " + f"{self.remote_host}:{remotepath}" - ), check=True) + ), cwd=self.appnamedir, shell=True, check=True) finally: pass From df079c91b524e18c8098fea54681daf58d374898 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:55:30 +0100 Subject: [PATCH 154/176] Correzione pubblicazione. --- loaih/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 0a5b820..9dcbdc0 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -410,10 +410,11 @@ class Build(loaih.RemoteBuild): else: remotepath = str.join('/', [ self.remote_path, '' ]) try: - subprocess.run(shlex.split( + subprocess.run( r"rsync -rlIvz --munge-links *.AppImage* " + - f"{self.remote_host}:{remotepath}" - ), cwd=self.appnamedir, shell=True, check=True) + f"{self.remote_host}:{remotepath}", + cwd=self.appnamedir, shell=True, check=True + ) finally: pass From 3aab2626edca32bd723f82191146623fc2c20fa2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:02:20 +0100 Subject: [PATCH 155/176] Correzione script. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 9dcbdc0..f00f999 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -387,7 +387,7 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), + checksum = subprocess.run(f"md5sum {file}", shell=True, capture_output=True, text=True, encoding='utf-8', check=True) if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: From f6bcf610ba2e1133a1c9a5dbdec923830ea60e37 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:07:03 +0100 Subject: [PATCH 156/176] Correzione md5. --- loaih/build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index f00f999..5649427 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -387,8 +387,10 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(f"md5sum {file}", shell=True, - capture_output=True, text=True, encoding='utf-8', check=True) + checksum = subprocess.run( + f"md5sum {file}", shell=True, cwd=self.appnamedir, + capture_output=True, text=True, encoding='utf-8', check=True + ) if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: csfile.write(checksum.stdout) From 701331818865924e5ebf2c26781b29075f244e83 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:15:42 +0100 Subject: [PATCH 157/176] Sostituito lancio programma con codice python. --- loaih/build.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 5649427..a99bc5b 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -10,6 +10,7 @@ import re import shlex import tempfile import urllib.request +import hashlib from lxml import etree import loaih @@ -387,13 +388,16 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run( - f"md5sum {file}", shell=True, cwd=self.appnamedir, - capture_output=True, text=True, encoding='utf-8', check=True - ) - if checksum.stdout: - with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: - csfile.write(checksum.stdout) + + checksum = hashlib.md5() + + with open(os.path.join(self.appnamedir, file), 'rb') as readfile: + for chunk in iter(lambda: readfile.read(4096), b""): + checksum.update(chunk) + + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.hexdigest()) + def publish(self): """Moves built versions to definitive storage.""" From 059518ccbf2f16d8f91ef3bb0b2803c81131e122 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:20:56 +0100 Subject: [PATCH 158/176] Ancora qualche correzione al codice per generare checksum. --- loaih/build.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index a99bc5b..80fda4b 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -382,21 +382,24 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) for arch in self.arch: for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - # For any built arch, find out if a file exist. - self.__create_checksum__(item) + if self.built[arch]: + # For any built arch, find out if a file exist. + self.__create_checksum__(item) def __create_checksum__(self, file): """Internal function to create checksum file.""" checksum = hashlib.md5() + fullpath = os.path.join(self.appnamedir, file) - with open(os.path.join(self.appnamedir, file), 'rb') as readfile: - for chunk in iter(lambda: readfile.read(4096), b""): - checksum.update(chunk) + if os.path.exists(fullpath): + with open(os.path.join(self.appnamedir, file), 'rb') as readfile: + for chunk in iter(lambda: readfile.read(4096), b""): + checksum.update(chunk) - with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: - csfile.write(checksum.hexdigest()) + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.hexdigest()) def publish(self): From 6bfa6a37079f7bbbc2e4c7a29ef4ca04ff07aea5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:24:35 +0100 Subject: [PATCH 159/176] Ancora correzioni al processo di checksum. --- loaih/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 80fda4b..0846dc3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -381,8 +381,8 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) for arch in self.arch: - for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - if self.built[arch]: + if self.built[arch]: + for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: # For any built arch, find out if a file exist. self.__create_checksum__(item) From dff74f0a35c330851774ec84f6f225edf510ab65 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:30:56 +0100 Subject: [PATCH 160/176] Ripristinate istruzioni originali, con chiamata a comando locale. --- loaih/build.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 0846dc3..6a8d479 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -390,16 +390,12 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = hashlib.md5() - fullpath = os.path.join(self.appnamedir, file) + checksum = subprocess.run(shlex.split(f"md5sum {file}"), + capture_output=True, text=True, encoding='utf-8', check=True) - if os.path.exists(fullpath): - with open(os.path.join(self.appnamedir, file), 'rb') as readfile: - for chunk in iter(lambda: readfile.read(4096), b""): - checksum.update(chunk) - - with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: - csfile.write(checksum.hexdigest()) + if checksum.stdout: + with open(f"{file}.md5", 'w', encoding='utf-8') as checkfile: + checkfile.write(checksum.stdout) def publish(self): From 2405601d2d5beaa5cd5fe694b04377fbd8f12131 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:46:45 +0100 Subject: [PATCH 161/176] Sistemato codice per build. --- loaih/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loaih/build.py b/loaih/build.py index 6a8d479..b73b4e7 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -364,6 +364,8 @@ class Build(loaih.RemoteBuild): ), env={ "VERSION": self.appversion }, check=True) print(f"Built AppImage version {self.appversion}") + # Setting it to built. + self.built[arch] = True # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): From 41dcbe1718f9b18302abe0d800ccbe297cd36253 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:52:21 +0100 Subject: [PATCH 162/176] Aggiunto codice per debug, corretto lancio comando di checksum. --- loaih/build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index b73b4e7..804fe0a 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -392,22 +392,26 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), - capture_output=True, text=True, encoding='utf-8', check=True) + checksum = subprocess.run(shlex.split(f"md5sum {file}"), shell=True, + capture_output=True, text=True, encoding='utf-8', check=True, + cwd=self.appnamedir) if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as checkfile: + print(f"DEBUG: writing checksum for {file}.") checkfile.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" + + print("DEBUG: reached publish.") + if all(self.built[arch] for arch in self.arch): # All files are already present in the full_path return os.chdir(self.appnamedir) - # Two cases here: local and remote storage_path. if self.remoterepo: # Remote first. From 5df2c5dbdb336da71119f6c59393c08220bbbab3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:59:38 +0100 Subject: [PATCH 163/176] Ancora sul build. Aggiunte stringhe per debugging. --- loaih/build.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 804fe0a..abfb12e 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -80,6 +80,9 @@ class Build(loaih.RemoteBuild): def calculate(self): """Calculate exclusions and other variables.""" + + print("--- Calculate Phase ---") + # AppName if self.query in { 'prerelease', 'daily' }: self.appname = 'LibreOfficeDev' @@ -132,6 +135,9 @@ class Build(loaih.RemoteBuild): def check(self): """Checking if the requested AppImage has been already built.""" + + print("--- Check Phase ---") + if len(self.appimagefilename) != 2: self.calculate() @@ -184,6 +190,9 @@ class Build(loaih.RemoteBuild): def download(self): """Downloads the contents of the URL as it was a folder.""" + + print("--- Download Phase ---") + print(f"Started downloads for {self.version}. Please wait.") for arch in self.arch: # Checking if a valid path has been provided @@ -227,6 +236,8 @@ class Build(loaih.RemoteBuild): def build(self): """Building all the versions.""" + print("--- Building Phase ---") + for arch in self.arch: if self.built[arch]: # Already built for arch or path not available. User has already been warned. @@ -364,8 +375,6 @@ class Build(loaih.RemoteBuild): ), env={ "VERSION": self.appversion }, check=True) print(f"Built AppImage version {self.appversion}") - # Setting it to built. - self.built[arch] = True # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): @@ -378,6 +387,9 @@ class Build(loaih.RemoteBuild): def checksums(self): """Create checksums of the built versions.""" # Skip checksum if initally the build was already found in the storage directory + + print("--- Checksum Phase ---") + if all(self.built[arch] for arch in self.arch): return @@ -386,6 +398,7 @@ class Build(loaih.RemoteBuild): if self.built[arch]: for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: # For any built arch, find out if a file exist. + print(f"DEBUG: checkumming {item}.") self.__create_checksum__(item) From 0a18586201c8e93c3245e4eb570730083694d62f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 01:03:21 +0100 Subject: [PATCH 164/176] Aggiunta testo per fasi. Rivista procedura. --- loaih/build.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index abfb12e..a0f021d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -395,11 +395,15 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) for arch in self.arch: - if self.built[arch]: + if not self.built[arch]: + # Here's the contrary. A newly built package has not yet been + # marked as built. for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - # For any built arch, find out if a file exist. - print(f"DEBUG: checkumming {item}.") - self.__create_checksum__(item) + itempath = os.path.join(self.appnamedir, item) + if os.path.exists(itempath): + # For any built arch, find out if a file exist. + print(f"DEBUG: checkumming {item}.") + self.__create_checksum__(item) def __create_checksum__(self, file): @@ -418,7 +422,7 @@ class Build(loaih.RemoteBuild): def publish(self): """Moves built versions to definitive storage.""" - print("DEBUG: reached publish.") + print("--- Publish Phase ---") if all(self.built[arch] for arch in self.arch): # All files are already present in the full_path @@ -454,6 +458,9 @@ class Build(loaih.RemoteBuild): def generalize_and_link(self, chdir = 'default'): """Creates the needed generalized files if needed.""" + + print("--- Generalize and Link Phase ---") + # If called with a pointed version, no generalize and link necessary. if not self.branch_version: return From c015aeea99ffaf11a53937ca3803bd9ecb255e2f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 01:25:32 +0100 Subject: [PATCH 165/176] Correzione codice checksum. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index a0f021d..356699d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -409,7 +409,7 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), shell=True, + checksum = subprocess.run(f"md5sum {file}", shell=True, capture_output=True, text=True, encoding='utf-8', check=True, cwd=self.appnamedir) From 03620cf013cd8306310d6ae1d850a47d3cad3b73 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 01:37:17 +0100 Subject: [PATCH 166/176] Correzione append percorsi. --- loaih/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 356699d..de67f36 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -433,10 +433,12 @@ class Build(loaih.RemoteBuild): if self.remoterepo: # Remote first. # Build destination directory - if len(self.relative_path) > 0: - remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) - else: + if len(self.relative_path) == 0: remotepath = str.join('/', [ self.remote_path, '' ]) + elif len(self.relative_path) == 1: + remotepath = str.join('/', [ self.remote_path, self.relative_path[0], '' ]) + else: + remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) try: subprocess.run( r"rsync -rlIvz --munge-links *.AppImage* " + From 0a3f475fa6c1c3017f67b8121de72144c093ff15 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 02:03:31 +0100 Subject: [PATCH 167/176] =?UTF-8?q?script=20=C3=A8=20linted.=20Corretto=20?= =?UTF-8?q?codice=20in=20check=20su=20build.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loaih/build.py | 9 ++++-- loaih/script.py | 82 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index de67f36..adb6cb0 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -149,10 +149,13 @@ class Build(loaih.RemoteBuild): # Remote storage. I have to query a remote site to know if it # was already built. name = self.appimagefilename[arch] - if len(self.relative_path) > 0: - path_arr = self.relative_path.insert(0, self.storage_path) - else: + if len(self.relative_path) == 0: path_arr = [ self.storage_path, '' ] + elif len(self.relative_path) == 1: + path_arr = [ self.storage_path, self.relative_path[0], '' ] + else: + path_arr = self.relative_path.insert(0, self.storage_path) + path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") matching = [] diff --git a/loaih/script.py b/loaih/script.py index bb1d7fe..29a6607 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -1,67 +1,85 @@ #!/usr/bin/env python # encoding: utf-8 +"""Helps with command line commands.""" +import json import click import yaml -import loaih, loaih.build -import re, sys, json +import loaih +import loaih.build @click.group() def cli(): - pass + """Helps with command line commands.""" @cli.command() -@click.option('-j', '--json', 'jsonout', default=False, is_flag=True, help="Output format in json.") +@click.option('-j', '--json', 'jsonout', default=False, is_flag=True, + help="Output format in json.") @click.argument('query') def getversion(query, jsonout): - b = [] + """Get the numeral version from a named version.""" + + batch = [] queries = [] if ',' in query: queries.extend(query.split(',')) else: queries.append(query) - for q in queries: - b.extend(loaih.Base.collectedbuilds(q)) + for singlequery in queries: + batch.extend(loaih.Base.collectedbuilds(singlequery)) - if len(b) > 0: + if len(batch) > 0: if jsonout: - click.echo(json.dumps([x.todict() for x in b])) + click.echo(json.dumps([x.todict() for x in batch])) else: - for v in b: - click.echo(v) + for value in batch: + click.echo(value) @cli.command() -@click.option('-a', '--arch', 'arch', type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), default='all', help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") -@click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") -@click.option('-d', '--download-path', 'download_path', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") -@click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") -@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") -@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") -@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") -@click.option('-s/-S', '--sign/--no-sign', 'sign', default=True, help="Wether to sign the build. Default: sign") -@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") +@click.option('-a', '--arch', 'arch', default='all', + type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), + help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") +@click.option('-c/-C', '--check/--no-check', 'check', default=True, + help="Check in the final storage if the queried version is existent. Default: check") +@click.option('-d', '--download-path', 'download_path', + default = '/var/tmp/downloads', type=str, + help="Path to the download folder. Default: /var/tmp/downloads") +@click.option('-l', '--language', 'language', default = 'basic', type=str, + help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") +@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, + help="Include or not the offline help for the chosen languages. Default: no offline help") +@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, + help="Create a portable version of the AppImage or not. Default: no portable") +@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', + type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=True, + help="Wether to sign the build. Default: sign") +@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, + help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): + """Builds an Appimage with the provided options.""" + # Parsing options arches = [] if arch.lower() == 'all': # We need to build it twice. - arches = [ u'x86', u'x86_64' ] + arches = [ 'x86', 'x86_64' ] else: arches = [ arch.lower() ] if query.endswith('.yml') or query.endswith('.yaml'): # This is a buildfile. So we have to load the file and pass the build options ourselves. config = {} - with open(query, 'r') as file: + with open(query, 'r', encoding= 'utf-8') as file: config = yaml.safe_load(file) # With the config file, we ignore all the command line options and set # generic default. - for build in config['builds']: + for cbuild in config['builds']: # Loop a run for each build. - collection = loaih.build.Collection(build['query'], arches) + collection = loaih.build.Collection(cbuild['query'], arches) for obj in collection: # Configuration phase @@ -69,12 +87,20 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.offline_help = build['offline_help'] obj.portable = build['portable'] obj.updatable = True - obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' - obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + obj.storage_path = "/srv/http/appimage.sys42.eu" + if 'repo' in config['data'] and config['data']['repo']: + obj.storage_path = config['data']['repo'] + obj.download_path = "/var/tmp/downloads" + if 'download' in config['data'] and config['data']['download']: + obj.download_path = config['data']['download'] if 'http' in obj.storage_path: obj.remoterepo = True - obj.remote_host = config['data']['remote_host'] if 'remote_host' in config['data'] and config['data']['remote_host'] else 'ciccio.libreitalia.org' - obj.remote_path = config['data']['remote_path'] if 'remote_path' in config['data'] and config['data']['remote_path'] else '/var/lib/nethserver/vhost/appimages' + obj.remote_host = "ciccio.libreitalia.org" + if 'remote_host' in config['data'] and config['data']['remote_host']: + obj.remote_host = config['data']['remote_host'] + obj.remote_path = "/var/lib/nethserver/vhost/appimages" + if 'remote_path' in config['data'] and config['data']['remote_path']: + obj.remote_path = config['data']['remote_path'] if 'sign' in config['data'] and config['data']['sign']: obj.sign = True From 64effab3d71eda9d93bb14d2dbd76cbc10f259f1 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 02:06:05 +0100 Subject: [PATCH 168/176] Corrette tutte le occorrenze di variabile rinominata. --- loaih/script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loaih/script.py b/loaih/script.py index 29a6607..4a4ed91 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -83,9 +83,9 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path for obj in collection: # Configuration phase - obj.language = build['language'] - obj.offline_help = build['offline_help'] - obj.portable = build['portable'] + obj.language = cbuild['language'] + obj.offline_help = cbuild['offline_help'] + obj.portable = cbuild['portable'] obj.updatable = True obj.storage_path = "/srv/http/appimage.sys42.eu" if 'repo' in config['data'] and config['data']['repo']: From db01651251e7bc574537d403d7459e244ef86bf5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 02:23:34 +0100 Subject: [PATCH 169/176] Gestito errore di verifica delle build con cartella quando non trovate. --- loaih/build.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index adb6cb0..29218e3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -159,14 +159,19 @@ class Build(loaih.RemoteBuild): path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") matching = [] - with urllib.request.urlopen(path) as url: - matching = etree.HTML(url.read()).xpath( - f"//a[contains(@href, '{name}')]/@href" - ) + try: + with urllib.request.urlopen(path) as url: + matching = etree.HTML(url.read()).xpath( + f"//a[contains(@href, '{name}')]/@href" + ) - if len(matching) > 0: - # Already built. - self.built[arch] = True + if len(matching) > 0: + # Already built. + self.built[arch] = True + + except urllib.error.HTTPError: + # The URL specified do not exist. So it is to build. + pass else: # Repo is local From 8c3e649a2527f8da842ca6ef0c304a96dc4b10b3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 21:46:40 +0100 Subject: [PATCH 170/176] Correzione file yaml per le build. --- daily.yml | 74 ++++++++++++++++++++++++++------------------------ fresh.yml | 74 ++++++++++++++++++++++++++------------------------ prerelease.yml | 74 ++++++++++++++++++++++++++------------------------ still.yml | 74 ++++++++++++++++++++++++++------------------------ 4 files changed, 152 insertions(+), 144 deletions(-) diff --git a/daily.yml b/daily.yml index 2f2f521..09d5573 100644 --- a/daily.yml +++ b/daily.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: daily language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: daily language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: daily language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: daily language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: daily language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: daily language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: daily language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: daily language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: daily language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: daily language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: daily language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: daily language: full - offline_help: yes - portable: yes + offline_help: true + portable: true diff --git a/fresh.yml b/fresh.yml index e8ba85d..2c196e1 100644 --- a/fresh.yml +++ b/fresh.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: fresh language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: fresh language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: fresh language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: fresh language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: fresh language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: fresh language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: fresh language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: fresh language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: fresh language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: fresh language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: fresh language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: fresh language: full - offline_help: yes - portable: yes + offline_help: true + portable: true diff --git a/prerelease.yml b/prerelease.yml index 402f385..37e0e04 100644 --- a/prerelease.yml +++ b/prerelease.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: prerelease language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: prerelease language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: prerelease language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: prerelease language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: prerelease language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: prerelease language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: prerelease language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: prerelease language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: prerelease language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: prerelease language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: prerelease language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: prerelease language: full - offline_help: yes - portable: yes + offline_help: true + portable: true diff --git a/still.yml b/still.yml index 3f5a595..6fdb968 100644 --- a/still.yml +++ b/still.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: still language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: still language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: still language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: still language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: still language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: still language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: still language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: still language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: still language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: still language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: still language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: still language: full - offline_help: yes - portable: yes + offline_help: true + portable: true From 78a43350ed22b1fb68c7c7ee6e8bb780420a9108 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 21:48:24 +0100 Subject: [PATCH 171/176] Cambio versione loaih. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c41059f..23029eb 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,13 @@ #!/usr/bin/env python # encoding: utf-8 # vim:sts=4:sw=4 +"""Helps building and automatizing building LibreOffice AppImages.""" from setuptools import setup,find_packages setup( name="loaih", - version="1.2.0", + version="1.3.0", description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", author="Emiliano Vavassori", author_email="syntaxerrormmm@libreoffice.org", From 0bea7a81bc7120a748c2fa0e1a19e8136504f158 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 22:17:18 +0100 Subject: [PATCH 172/176] Correzione di un paio di errori di sintassi sui percorsi multipli. --- loaih/build.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 29218e3..b42c9ff 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -154,10 +154,10 @@ class Build(loaih.RemoteBuild): elif len(self.relative_path) == 1: path_arr = [ self.storage_path, self.relative_path[0], '' ] else: - path_arr = self.relative_path.insert(0, self.storage_path) + path_arr = self.relative_path + path_arr.insert(0, self.storage_path) path = str.join('/', path_arr) - print(f"DEBUG - Name: {name}, URL: {path}") matching = [] try: with urllib.request.urlopen(path) as url: @@ -410,7 +410,6 @@ class Build(loaih.RemoteBuild): itempath = os.path.join(self.appnamedir, item) if os.path.exists(itempath): # For any built arch, find out if a file exist. - print(f"DEBUG: checkumming {item}.") self.__create_checksum__(item) @@ -423,7 +422,6 @@ class Build(loaih.RemoteBuild): if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as checkfile: - print(f"DEBUG: writing checksum for {file}.") checkfile.write(checksum.stdout) @@ -446,7 +444,9 @@ class Build(loaih.RemoteBuild): elif len(self.relative_path) == 1: remotepath = str.join('/', [ self.remote_path, self.relative_path[0], '' ]) else: - remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) + path_arr = self.relative_path + path_arr.insert(0, self.remote_path) + remotepath = str.join('/', path_arr) try: subprocess.run( r"rsync -rlIvz --munge-links *.AppImage* " + From 8b5c87f80170f814cad0cc9de6510d2bbc898038 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 22:52:11 +0100 Subject: [PATCH 173/176] Correzioni su full path. --- loaih/build.py | 59 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index b42c9ff..9116ad9 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -47,7 +47,7 @@ class Build(loaih.RemoteBuild): self.portable = False self.updatable = True self.sign = True - self.remoterepo = False + self.repo_type = 'local' self.remote_host = '' self.remote_path = '' self.storage_path = '/mnt/appimage' @@ -126,11 +126,22 @@ class Build(loaih.RemoteBuild): if self.portable: self.relative_path.append('portable') - fullpath_arr = self.storage_path.split('/') - # Joining relative path only if it is not null - if len(self.relative_path) > 0: - fullpath_arr.extend(self.relative_path) - self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) + # Fullpath might be intended two ways: + if self.repo_type == 'remote': + # Repository is remote + # we build full_path as it is absolute to the root of the + # storage_path. + self.full_path = '/' + if len(self.relative_path) >= 1: + self.full_path += str.join('/', self.relative_path) + else: + # Repository is local + # If it is remote or if it is local + fullpath_arr = self.storage_path.split('/') + # Joining relative path only if it is not null + if len(self.relative_path) > 0: + fullpath_arr.extend(self.relative_path) + self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) def check(self): @@ -138,30 +149,27 @@ class Build(loaih.RemoteBuild): print("--- Check Phase ---") + # let's check here if we are on a remote repo or local. + if self.storage_path.startswith("http"): + # Final repository is remote + self.repo_type = 'remote' + else: + self.repo_type = 'local' + if len(self.appimagefilename) != 2: self.calculate() for arch in self.arch: - # First, check if by metadata the repo is remote or not. - if self.remoterepo or 'http' in self.storage_path: - self.remoterepo = True + if self.repo_type == 'remote': # Remote storage. I have to query a remote site to know if it # was already built. name = self.appimagefilename[arch] - if len(self.relative_path) == 0: - path_arr = [ self.storage_path, '' ] - elif len(self.relative_path) == 1: - path_arr = [ self.storage_path, self.relative_path[0], '' ] - else: - path_arr = self.relative_path - path_arr.insert(0, self.storage_path) - - path = str.join('/', path_arr) + url = self.storage_path.rstrip('/') + self.full_path + '/' + name matching = [] try: - with urllib.request.urlopen(path) as url: - matching = etree.HTML(url.read()).xpath( + with urllib.request.urlopen(url) as response: + matching = etree.HTML(response.read()).xpath( f"//a[contains(@href, '{name}')]/@href" ) @@ -436,17 +444,10 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) # Two cases here: local and remote storage_path. - if self.remoterepo: + if self.repo_type == 'remote': # Remote first. # Build destination directory - if len(self.relative_path) == 0: - remotepath = str.join('/', [ self.remote_path, '' ]) - elif len(self.relative_path) == 1: - remotepath = str.join('/', [ self.remote_path, self.relative_path[0], '' ]) - else: - path_arr = self.relative_path - path_arr.insert(0, self.remote_path) - remotepath = str.join('/', path_arr) + remotepath = self.remote_path.rstrip('/') + self.full_path try: subprocess.run( r"rsync -rlIvz --munge-links *.AppImage* " + From 1f83db6105678aaf9a40cbf08c5bb5be7b17db9f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 22:57:15 +0100 Subject: [PATCH 174/176] Ristrutturazione codice di verifica. --- loaih/build.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 9116ad9..e06a2b4 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -82,7 +82,16 @@ class Build(loaih.RemoteBuild): """Calculate exclusions and other variables.""" print("--- Calculate Phase ---") - + + # let's check here if we are on a remote repo or local. + if self.storage_path.startswith("http"): + # Final repository is remote + self.repo_type = 'remote' + print("Repo is remote.") + else: + self.repo_type = 'local' + print("Repo is local.") + # AppName if self.query in { 'prerelease', 'daily' }: self.appname = 'LibreOfficeDev' @@ -149,13 +158,6 @@ class Build(loaih.RemoteBuild): print("--- Check Phase ---") - # let's check here if we are on a remote repo or local. - if self.storage_path.startswith("http"): - # Final repository is remote - self.repo_type = 'remote' - else: - self.repo_type = 'local' - if len(self.appimagefilename) != 2: self.calculate() From 3a9f13594c62c621fb5b209f4eff46abd8dc1b8f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 23:08:33 +0100 Subject: [PATCH 175/176] Fix probabilmente definitivo. --- loaih/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index e06a2b4..8bf07fb 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -10,7 +10,6 @@ import re import shlex import tempfile import urllib.request -import hashlib from lxml import etree import loaih @@ -167,7 +166,7 @@ class Build(loaih.RemoteBuild): # Remote storage. I have to query a remote site to know if it # was already built. name = self.appimagefilename[arch] - url = self.storage_path.rstrip('/') + self.full_path + '/' + name + url = self.storage_path.rstrip('/') + self.full_path + '/' matching = [] try: with urllib.request.urlopen(url) as response: From b087e85ec5c3abfb1a0049065e2bb013e56b08aa Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 23:22:35 +0100 Subject: [PATCH 176/176] Semplificati alcuni messaggi nell'output. --- loaih/build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 8bf07fb..0c619d5 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -161,6 +161,7 @@ class Build(loaih.RemoteBuild): self.calculate() for arch in self.arch: + print(f"Searching for {self.appimagefilename[arch]}") # First, check if by metadata the repo is remote or not. if self.repo_type == 'remote': # Remote storage. I have to query a remote site to know if it @@ -184,7 +185,6 @@ class Build(loaih.RemoteBuild): else: # Repo is local - print(f"Searching for {self.appimagefilename[arch]}") command = f"find {self.full_path} -name {self.appimagefilename[arch]}" res = subprocess.run(shlex.split(command), capture_output=True, @@ -198,11 +198,10 @@ class Build(loaih.RemoteBuild): if res.stdout and len(res.stdout.strip("\n")) > 0: # All good, the command was executed fine. - print(f"Build for {self.version} found.") self.built[arch] = True if self.built[arch]: - print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + print(f"Found requested AppImage: {self.appimagefilename[arch]}.") def download(self): @@ -214,7 +213,7 @@ class Build(loaih.RemoteBuild): for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': - print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.") + print(f"Cannot build for arch {arch}. Continuing with other arches.") # Faking already built it so to skip other checks. self.built[arch] = True continue