diff --git a/loaih/__init__.py b/loaih/__init__.py index a612057..a754aff 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -3,7 +3,7 @@ import urllib.request import loaih.versions as versions from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re, shlex +import tempfile, os, sys, glob, subprocess, shutil, re class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] @@ -13,35 +13,32 @@ class Build(object): def __init__(self, query, arch): """Build all versions that can be found in the indicated repo.""" self.query = query + self.queried_name = False if '.' in self.query else True self.arch = arch - - # Getting versions and so on - v = versions.BuildVersion(self.query) - self.version = v.version - print(f"Debug {self.version}") - self.short_version = str.join('.', self.version.split('.')[0:2]) - self.branch_version = None - if not '.' in self.query: - self.branch_version = self.query - self.url = v.basedirurl - - # Other default values + self.url = {} self.language = 'basic' self.offline_help = False self.portable = False self.updatable = True - self.sign = True - self.storage_path = '/mnt/appimage' + self.sign = False + self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' # Specific build version self.appversion = '' + self.genappversion = '' self.appimagefilename = {} - self.zsyncfilename = {} + self.genappimagefilename = {} + + # Getting versions and so on + v = versions.BuildVersion(self.query) # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} + self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' + self.version = v.version + self.url = v.basedirurl self.built = { u'x86': False, u'x86_64': False } # Preparing the default for the relative path on the storage for @@ -50,38 +47,77 @@ class Build(object): # understood the storage_path can be changed before that phase. self.relative_path = [] self.full_path = '' - self.baseurl = '' def calculate(self): """Calculate exclusions and other variables.""" - # AppName - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' + # Incompatibilities - if portable and updatable are asked together, + # only portable will be built. + if self.portable and self.updatable: + print("Upgradable and portable options were required together. Building only portable.") + self.updatable = False - # Calculating languagepart + if self.updatable and not self.queried_name: + # If the queried version was a numbered version, doesn't make sense + # to build an updatable version. + self.updatable = False + + # Mandate to the private function to calculate the full_path available + # for the storage and the checks. + self.__calculate_full_path__() + + # Building expected AppImageName self.languagepart = "." if ',' in self.language: self.languagepart += self.language.replace(',', '-') else: self.languagepart += self.language - # Calculating help part self.helppart = '.help' if self.offline_help else '' - # Building the required names + # If the build was called by queried name, build from latest release available but build with the most generic name + self.appversion = self.version + self.languagepart + self.helppart + myver = str.join('.', self.version.split('.')[0:2]) + self.genappversion = myver + self.languagepart + self.helppart for arch in Build.ARCHSTD: - self.appimagefilename[arch] = self.__gen_appimagefilename__(self.version, arch) - self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' - - # Mandate to the private function to calculate the full_path available - # for the storage and the checks. - self.__calculate_full_path__() + self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' + self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' - def __gen_appimagefilename__(self, version, arch): - """Generalize the construction of the name of the app.""" - self.appversion = version + self.languagepart + self.helppart - return self.appname + f'-{self.appversion}-{arch}.AppImage' + def check(self): + """Checking if the requested AppImage has been already built.""" + + for arch in self.arch: + # For generalized builds, we need to check if there are .ver file + # and it contains the specific version found. + print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) + res = subprocess.run("find {path} -name {appimage}'".format( + path = self.full_path, + appimage = self.genappimagefilename[arch] + '.ver' + ), shell=True, capture_output=True, env={ "LC_ALL": "C" }) + + if "No such file or directory" in res.stderr.decode('utf-8'): + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout: + # All good, the command was executed fine. + for file in res.stdout.decode('utf-8').strip('\n').split('\n'): + if self.version in open(file, 'r').read(): + self.built[arch] = True + + print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) + res = subprocess.run("find {path} -name '{appimage}'".format( + path = self.full_path, + appimage = self.appimagefilename[arch] + ), shell=True, capture_output=True) + if res.stdout: + if len(res.stdout.decode('utf-8').strip('\n')) > 1: + self.built[arch] = True + + if self.built[arch]: + print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) def __calculate_full_path__(self): @@ -89,7 +125,7 @@ class Build(object): if len(self.relative_path) == 0: if self.query == 'daily': self.relative_path.append('daily') - elif self.query == 'primageerelease': + elif self.query == 'prerelease': self.relative_path.append('prerelease') # Not the same check, an additional one @@ -103,68 +139,49 @@ class Build(object): self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) - def check(self): - """Checking if the requested AppImage has been already built.""" - if not len(self.appimagefilename) == 2: - self.calculate() - - for arch in self.arch: - print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') - - if "No such file or directory" in res.stderr: - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue - - if res.stdout and len(res.stdout.strip("\n")) > 0: - # All good, the command was executed fine. - print(f"Build for {self.version} found.") - self.built[arch] = True - - if self.built[arch]: - print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") - - def download(self): """Downloads the contents of the URL as it was a folder.""" - print(f"Started downloads for {self.version}. Please wait.") for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': - print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.") + print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) # Faking already built it so to skip other checks. self.built[arch] = True continue if self.built[arch]: - print(f"A build for {arch} was already found. Skipping specific packages.") + print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) continue - # Identifying downloads contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] tarballs = self.tarballs[arch] maintarball = tarballs[0] - # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) for archive in tarballs: # If the archive is already there, do not do anything. - if os.path.exists(archive): + if os.path.exists(os.path.join(self.download_path, archive)): + print("Archive %s is already there! Sweet" % archive) continue # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) except: - print(f"Failed to download {archive}.") + print("Failed to download {archive}.".format(archive = archive)) + + print("Got %s." % archive) - print(f"Finished downloads for {self.version}.") def build(self): """Building all the versions.""" + # We have 4 builds to do: + # * standard languages, no help + # * standard languages + offline help + # * all languages, no help + # * all languages + offline help for arch in self.arch: if self.built[arch]: @@ -173,19 +190,28 @@ class Build(object): # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) - os.makedirs(self.appnamedir, exist_ok=True) + self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') + os.makedirs(self.appimagedir, exist_ok = True) # And then cd to the appname folder. os.chdir(self.appnamedir) # Download appimagetool from github - appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage" + appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') os.chmod('appimagetool', 0o755) # Build the requested version. + if self.queried_name and not self.portable: + # If it is portable, do not generate a generalized version + self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) - def __unpackbuild__(self, arch): + def __unpackbuild__(self, arch, generalize = False): + if generalize and self.portable: + # Doesn't particularly make sense to build a generic portable + # version. Just skipping the specific generic build + return + # We start by filtering out tarballs from the list buildtarballs = [ self.tarballs[arch][0] ] @@ -217,46 +243,35 @@ class Build(object): else: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - os.chdir(self.appnamedir) - # Unpacking the tarballs for archive in buildtarballs: - subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) + + os.chdir(self.appnamedir) - # create appimagedir - self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) - # At this point, let's decompress the deb packages - subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) + subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) if self.portable: - subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) + shortversion = str.join('.', self.version.split('.')[:3]) + subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) # Changing desktop file - subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) - subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) + subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) - subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) # Find the name of the binary called in the desktop file. - binaryname = '' - with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: - a = d.readlines() - for line in a: - if re.match(r'^Exec', line): - binaryname = line.split('=')[-1].split(' ')[0] - # Esci al primo match - break - #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') - #binaryname = binary_exec.stdout.strip("\n") + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) + subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) # Download AppRun from github - apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" + apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) @@ -265,42 +280,58 @@ class Build(object): buildopts = [] if self.sign: buildopts.append('--sign') - - # adding zsync build if updatable - if self.updatable: - buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'") - - buildopts_str = str.join(' ', buildopts) - # Build the number-specific build - subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) - print(f"Built AppImage version {self.appversion}") + # Building app + if self.updatable: + # Updatable make sense only for generic images for fresh, still, + # daily. If a request was for a specific version, I'd not build an + # updatable version. + # zsync name was generated already + + # If asked to do a generalized build: + if generalize: + subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + # Build version file management + with open(self.genappimagefilename[arch] + '.ver', 'w') as v: + v.write(self.version) + else: + subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) + + else: + if generalize: + subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) + with open(self.genappimagefilename[arch] + '.ver', 'w') as v: + v.write(self.version) + else: + subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) + + print("Built AppImage version {version}".format(version = self.appversion)) # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) + subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) def checksums(self): """Create checksums of the built versions.""" - # Skip checksum if initally the build was already found in the storage directory if all(self.built.values()): - return + # All checksums are already created. + return + + # On the contrary, checksums will be in any case overwritten if + # existent, but generated only for built packages anyways os.chdir(self.appnamedir) - for arch in self.arch: - for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - # For any built arch, find out if a file exist. - self.__create_checksum__(item) + for appimage in glob.glob('*.AppImage*'): + if appimage.endswith('.ver'): + # Skipping checksums for .ver files. + continue + # See if a checksum already exist + if not os.path.exists(appimage + '.md5'): + subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) - def __create_checksum__(self, file): - """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') - if checksum.stdout: - with open(f"{file}.md5", 'w') as c: - c.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" @@ -311,54 +342,7 @@ class Build(object): os.chdir(self.appnamedir) # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) - for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - - - def generalize_and_link(self): - """Creates the needed generalized files if needed.""" - # If called with a pointed version, no generalize and link necessary. - if not self.branch_version: - return - appimagefilename = {} - zsyncfilename = {} - - # Creating versions for short version and query text - versions = [ self.short_version, self.branch_version ] - for arch in Build.ARCHSTD: - # If already built, do not do anything. - if self.built[arch]: - continue - - os.chdir(self.full_path) - # if the appimage for the reported arch is not found, skip to next - # arch - if not os.path.exists(self.appimagefilename[arch]): - continue - - # Doing it both for short_name and for branchname - for version in versions: - appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' - zsyncfilename[arch] = appimagefilename[arch] + '.zsync' - - # Create the symlink - print(f"Creating {appimagefilename[arch]} and checksums.") - if os.path.exists(appimagefilename[arch]): - os.unlink(appimagefilename[arch]) - os.symlink(self.appimagefilename[arch], appimagefilename[arch]) - # Create the checksum for the AppImage - self.__create_checksum__(appimagefilename[arch]) - # Do not continue if no zsync file is provided. - if not self.updatable: - continue - - print(f"Creating zsync file for version {version}.") - if os.path.exists(zsyncfilename[arch]): - os.unlink(zsyncfilename[arch]) - shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) - # Editing the zsyncfile - subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) - self.__create_checksum__(zsyncfilename[arch]) + subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) def __del__(self): diff --git a/loaih/versions.py b/loaih/versions.py index 201e531..fa0d972 100644 --- a/loaih/versions.py +++ b/loaih/versions.py @@ -6,7 +6,6 @@ from lxml import etree from packaging.version import parse as parse_version class BuildVersion(object): - DOWNLOADPAGE = "https://www.libreoffice.org/download/download/" ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" @@ -68,23 +67,13 @@ class BuildVersion(object): return { 'version': version, 'basedirurl': basedirurl } # Stable releases. - # Old approach - Doesn't really work because RelEng can screw order. - #versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') - #index = 1 - #if branch == 'still': - # index = -2 - #elif branch == 'fresh': - # index = -1 - #version = self.__getlatestrel(versions[index].text.strip('/')) - - # Now I'll rely on DownloadPage - versions = etree.HTML(urllib.request.urlopen(BuildVersion.DOWNLOADPAGE).read()).xpath('//span[@class="dl_version_number"]') - index = 0 + versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') + index = 1 if branch == 'still': - index = 1 + index = -2 elif branch == 'fresh': - index = 0 - version = self.__getlatestrel(versions[index].text) + index = -1 + version = self.__getlatestrel(versions[index].text.strip('/')) return { 'version': version, 'basedirurl': self.__getbaseurl(version) } diff --git a/scripts/loaih-build b/scripts/loaih-build index 94cda81..7a1ead4 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -12,8 +12,8 @@ import loaih @click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") -@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") -@click.option('-s/-S', '--sign/--no-sign', 'sign', default=True, help="Wether to sign the build. Default: sign") +@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): @@ -57,7 +57,6 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.build() obj.checksums() obj.publish() - obj.generalize_and_link() del obj else: @@ -84,7 +83,6 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.build() obj.checksums() obj.publish() - obj.generalize_and_link() del obj if __name__ == '__main__': diff --git a/test.yml b/test.yml deleted file mode 100644 index faf4464..0000000 --- a/test.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -data: - repo: /mnt/appimage - download: /var/tmp/downloads - force: yes - sign: yes - -builds: - - query: fresh - language: basic - offline_help: no - portable: no