diff --git a/loaih/__init__.py b/loaih/__init__.py index a754aff..3c20115 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -3,7 +3,7 @@ import urllib.request import loaih.versions as versions from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re +import tempfile, os, sys, glob, subprocess, shutil, re, shlex class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] @@ -13,32 +13,34 @@ class Build(object): def __init__(self, query, arch): """Build all versions that can be found in the indicated repo.""" self.query = query - self.queried_name = False if '.' in self.query else True self.arch = arch - self.url = {} + + # Getting versions and so on + v = versions.BuildVersion(self.query) + self.version = v.version + self.short_version = str.join('.', self.version.split('.')[0:2]) + self.branch_version = None + if not '.' in self.query: + self.branch_version = self.query + self.url = v.basedirurl + + # Other default values self.language = 'basic' self.offline_help = False self.portable = False self.updatable = True - self.sign = False - self.storage_path = '/srv/http/appimage.sys42.eu' + self.sign = True + self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' # Specific build version self.appversion = '' - self.genappversion = '' self.appimagefilename = {} - self.genappimagefilename = {} - - # Getting versions and so on - v = versions.BuildVersion(self.query) + self.zsyncfilename = {} # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' - self.version = v.version - self.url = v.basedirurl self.built = { u'x86': False, u'x86_64': False } # Preparing the default for the relative path on the storage for @@ -51,73 +53,32 @@ class Build(object): def calculate(self): """Calculate exclusions and other variables.""" - # Incompatibilities - if portable and updatable are asked together, - # only portable will be built. - if self.portable and self.updatable: - print("Upgradable and portable options were required together. Building only portable.") - self.updatable = False + # AppName + self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' - if self.updatable and not self.queried_name: - # If the queried version was a numbered version, doesn't make sense - # to build an updatable version. - self.updatable = False - - # Mandate to the private function to calculate the full_path available - # for the storage and the checks. - self.__calculate_full_path__() - - # Building expected AppImageName + # Calculating languagepart self.languagepart = "." if ',' in self.language: self.languagepart += self.language.replace(',', '-') else: self.languagepart += self.language + # Calculating help part self.helppart = '.help' if self.offline_help else '' - # If the build was called by queried name, build from latest release available but build with the most generic name - self.appversion = self.version + self.languagepart + self.helppart - myver = str.join('.', self.version.split('.')[0:2]) - self.genappversion = myver + self.languagepart + self.helppart + # Building the required names for arch in Build.ARCHSTD: - self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' - self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' + self.appimagefilename[arch] = __genappimagefilename__(self.version, arch) + self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync' + + # Mandate to the private function to calculate the full_path available + # for the storage and the checks. + self.__calculate_full_path__() - def check(self): - """Checking if the requested AppImage has been already built.""" - - for arch in self.arch: - # For generalized builds, we need to check if there are .ver file - # and it contains the specific version found. - print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) - res = subprocess.run("find {path} -name {appimage}'".format( - path = self.full_path, - appimage = self.genappimagefilename[arch] + '.ver' - ), shell=True, capture_output=True, env={ "LC_ALL": "C" }) - - if "No such file or directory" in res.stderr.decode('utf-8'): - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue - - if res.stdout: - # All good, the command was executed fine. - for file in res.stdout.decode('utf-8').strip('\n').split('\n'): - if self.version in open(file, 'r').read(): - self.built[arch] = True - - print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) - res = subprocess.run("find {path} -name '{appimage}'".format( - path = self.full_path, - appimage = self.appimagefilename[arch] - ), shell=True, capture_output=True) - if res.stdout: - if len(res.stdout.decode('utf-8').strip('\n')) > 1: - self.built[arch] = True - - if self.built[arch]: - print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) + def __gen_appimagefilename__(self, version, arch): + """Generalize the construction of the name of the app.""" + return self.appname + f'-{version}' + self.languagepart + self.helppart + f'-{arch}.AppImage' def __calculate_full_path__(self): @@ -139,49 +100,70 @@ class Build(object): self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) + def check(self): + """Checking if the requested AppImage has been already built.""" + if not len(self.appimagefilename) == 2: + self.calculate() + + for arch in self.arch: + print(f"Searching for {self.appimagefilename[arch]}") + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}'"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + + if "No such file or directory" in res.stderr: + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout: + # All good, the command was executed fine. + for file in res.stdout.strip('\n').split('\n'): + if self.version in open(file, 'r').read(): + print(f"Build for {self.version} found.") + self.built[arch] = True + + if self.built[arch]: + print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + + def download(self): """Downloads the contents of the URL as it was a folder.""" + print(f"Started downloads for {self.version}. Please wait.") for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': - print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) + print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.") # Faking already built it so to skip other checks. self.built[arch] = True continue if self.built[arch]: - print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) + print(f"A build for {arch} was already found. Skipping specific packages.") continue + # Identifying downloads contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] tarballs = self.tarballs[arch] maintarball = tarballs[0] + # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) for archive in tarballs: # If the archive is already there, do not do anything. - if os.path.exists(os.path.join(self.download_path, archive)): - print("Archive %s is already there! Sweet" % archive) + if os.path.exists(archive): continue # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) except: - print("Failed to download {archive}.".format(archive = archive)) - - print("Got %s." % archive) + print(f"Failed to download {archive}.") + print(f"Finished downloads for {self.version}.") def build(self): """Building all the versions.""" - # We have 4 builds to do: - # * standard languages, no help - # * standard languages + offline help - # * all languages, no help - # * all languages + offline help for arch in self.arch: if self.built[arch]: @@ -190,28 +172,18 @@ class Build(object): # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) - self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') - os.makedirs(self.appimagedir, exist_ok = True) # And then cd to the appname folder. os.chdir(self.appnamedir) # Download appimagetool from github - appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) + appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage" urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') os.chmod('appimagetool', 0o755) # Build the requested version. - if self.queried_name and not self.portable: - # If it is portable, do not generate a generalized version - self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) - def __unpackbuild__(self, arch, generalize = False): - if generalize and self.portable: - # Doesn't particularly make sense to build a generic portable - # version. Just skipping the specific generic build - return - + def __unpackbuild__(self, arch): # We start by filtering out tarballs from the list buildtarballs = [ self.tarballs[arch][0] ] @@ -243,35 +215,38 @@ class Build(object): else: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - # Unpacking the tarballs - for archive in buildtarballs: - subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) - os.chdir(self.appnamedir) + # Unpacking the tarballs + for archive in buildtarballs: + subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + + # create appimagedir + self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) + # At this point, let's decompress the deb packages - subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) if self.portable: - shortversion = str.join('.', self.version.split('.')[:3]) - subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) # Changing desktop file - subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname), cwd=self.appimagedir) - subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) + subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) # Find the name of the binary called in the desktop file. - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') + binary_exec = subprocess.run(shlex.split("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') + binaryname = binary_exec.stdout.strip("\n") bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) + subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) # Download AppRun from github - apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) + apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) @@ -280,57 +255,36 @@ class Build(object): buildopts = [] if self.sign: buildopts.append('--sign') - - # Building app + + # adding zsync build if updatable if self.updatable: - # Updatable make sense only for generic images for fresh, still, - # daily. If a request was for a specific version, I'd not build an - # updatable version. - # zsync name was generated already + buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'") - # If asked to do a generalized build: - if generalize: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) - # Build version file management - with open(self.genappimagefilename[arch] + '.ver', 'w') as v: - v.write(self.version) - else: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) - - else: - if generalize: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) - with open(self.genappimagefilename[arch] + '.ver', 'w') as v: - v.write(self.version) - else: - subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True) - - print("Built AppImage version {version}".format(version = self.appversion)) + buildopts_str = str.join(' ', buildopts) + # Build the number-specific build + subprocess.run(shlex.split(f"VERSION={self.appversion} ./appimagetool {buildopts_str} -v ./{self.appname}.AppDir/")) + + print(f"Built AppImage version {self.appversion}") # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) + subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) def checksums(self): """Create checksums of the built versions.""" + # Skip checksum if initally the build was already found in the storage directory if all(self.built.values()): - # All checksums are already created. - return - - # On the contrary, checksums will be in any case overwritten if - # existent, but generated only for built packages anyways + return os.chdir(self.appnamedir) - for appimage in glob.glob('*.AppImage*'): - if appimage.endswith('.ver'): - # Skipping checksums for .ver files. - continue - - # See if a checksum already exist - if not os.path.exists(appimage + '.md5'): - subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) + for arch in self.arch: + for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: + # For any built arch, find out if a file exist. + if len(glob.glob(self.appimagefilename[arch] + '.md5')) == 0: + # Build checksum + subprocess.run(shlex.split(f"md5sum {item} > {item}.md5")) def publish(self): @@ -342,7 +296,41 @@ class Build(object): os.chdir(self.appnamedir) # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) - subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) + for file in glob.glob("*.AppImage*"): + subprocess.run(shlex.split(f"cp -f {file} {self.fullpath}")) + + + def generalize_and_link(self): + """Creates the needed generalized files if needed.""" + # If called with a pointed version, no generalize and link necessary. + if not self.branch_version: + return + + # Creating versions for short version and query text + versions = [ self.short_version, self.branch_version ] + for arch in Build.ARCHSTD: + # if the appimage for the reported arch is not found, skip to next + # arch + if not os.path.exists(self.appimagefilename[arch]): + continue + + # Doing it both for short_name and for branchname + for version in versions: + appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + zsyncfilename[arch] = appimagefilename[arch] + '.zsync' + + os.chdir(self.full_path) + # Create the symlink + os.symlink(self.appimagefilename[arch], appimagefilename[arch]) + # Create the checksum for the AppImage + subprocess.run(shlex.split("md5sum {item} > {item}.md5".format(item=appimagefilename[arch]))) + # Do not continue if no zsync file is provided. + if not self.updatable: + continue + + os.copy(self.zsyncfilename[arch], zsyncfilename[arch]) + # Editing the zsyncfile + subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) def __del__(self): diff --git a/scripts/loaih-build b/scripts/loaih-build index 7a1ead4..08c3892 100644 --- a/scripts/loaih-build +++ b/scripts/loaih-build @@ -12,7 +12,7 @@ import loaih @click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") -@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") +@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") @click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') @@ -57,6 +57,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.build() obj.checksums() obj.publish() + obj.generalize_and_link() del obj else: