diff --git a/.gitignore b/.gitignore index ff7d489..5ceb386 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ venv -build -dist -loaih.egg-info diff --git a/build.py b/build.py new file mode 100644 index 0000000..1dda163 --- /dev/null +++ b/build.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +import urllib.request +from lxml import etree +import tempfile, os, sys, glob, subprocess, shutil + +class Build(object): + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] + + def __init__(self, query, arch, url, downloaddir = '/var/tmp/downloads'): + """Build all versions that can be found in the indicated repo.""" + self.__query__ = query + self.__arch__ = arch + self.__url__ = url + self.__downloaddir__ = downloaddir + + # Creating a tempfile + self.__builddir__ = tempfile.mkdtemp() + self.__tarballs__ = [] + self.__appname__ = '' + self.__version__ = '' + + if self.__url__ == '-': + print("Cannot find this version for arch {arch}.".format(arch = self.__arch__)) + return False + + + def download(self): + """Downloads the contents of the URL as it was a folder.""" + + # Let's start with defining which files are to be downloaded. + # Let's explore the remote folder. + contents = etree.HTML(urllib.request.urlopen(self.__url__).read()).xpath("//td/a") + self.__tarballs__ = [ x.text for x in contents if x.text.endswith('tar.gz') ] + maintarball = self.__tarballs__[0] + main_arr = maintarball.split('_') + self.__appname__ = main_arr[0] + self.__version__ = main_arr[1] + + os.makedirs(self.__downloaddir__, exist_ok = True) + os.chdir(self.__downloaddir__) + for archive in self.__tarballs__: + # If the archive is already there, do not do anything. + if os.path.exists(os.path.join(self.__downloaddir__, archive)): + print("Archive %s is already there! Sweet" % archive) + continue + + # Download the archive + try: + urllib.request.urlretrieve(self.__url__ + archive, archive) + except: + print("Failed to download {archive}.".format(archive = archive)) + + print("Got %s." % archive) + + + def build(self): + """Building all the versions.""" + # We have 4 builds to do: + # * standard languages, no help + # * standard languages + offline help + # * all languages, no help + # * all languages + offline help + + # Preparation tasks + self.appnamedir = os.path.join(self.__builddir__, self.__appname__) + self.appimagedir = os.path.join(self.__builddir__, self.__appname__, self.__appname__ + '.AppDir') + os.makedirs(self.appimagedir, exist_ok = True) + # And then cd to the appname folder. + os.chdir(self.appnamedir) + # Download appimagetool from github + appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = self.__arch__) + urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') + os.chmod('appimagetool', 0o755) + + # Run to build standard no help + self.__unpackbuild__('standard', False) + + # Run to build standard full help + self.__unpackbuild__('standard', True) + + # Build full no help + self.__unpackbuild__('full', False) + + # Full with help + self.__unpackbuild__('full', True) + + def __unpackbuild__(self, languageset = 'full', offlinehelp = False): + # We start by filtering out tarballs from the list + buildtarballs = [ self.__tarballs__[0] ] + + # Let's process standard languages and append results to the + # buildtarball + if languageset == 'standard': + for lang in Build.LANGSTD: + buildtarballs.extend([ x for x in self.__tarballs__ if ('langpack_' + lang) in x ]) + if offlinehelp: + buildtarballs.extend([ x for x in self.__tarballs__ if ('helppack_' + lang) in x ]) + else: + # In any other cases, we build with all languages + if not offlinehelp: + buildtarballs.extend([ x for x in self.__tarballs__ if 'langpack_' in x ]) + else: + # We need also all help. Let's replace buildtarball with the + # whole bunch + buildtarballs = self.__tarballs__ + + # Unpacking the tarballs + for archive in buildtarballs: + subprocess.run("tar xzf {folder}/{archive}".format(folder = self.__downloaddir__, archive = archive), shell=True) + + os.chdir(self.appnamedir) + os.makedirs(self.appimagedir, exist_ok = True) + # At this point, let's decompress the deb packages + subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) + # Changing desktop file + subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) + appname = 'LibreOffice' if not self.__query__ == 'daily' else 'LibreOfficeDev' + subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % appname, shell=True, cwd=self.appimagedir) + + subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) + + # Find the name of the binary called in the desktop file. + binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') + + bindir=os.path.join(self.appimagedir, 'usr', 'bin') + os.makedirs(bindir, exist_ok = True) + subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) + + # Download AppRun from github + apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = self.__arch__) + dest = os.path.join(self.appimagedir, 'AppRun') + urllib.request.urlretrieve(apprunurl, dest) + os.chmod(dest, 0o755) + + # Setting app version + appversion = self.__version__ + '.' + languageset + if offlinehelp: + appversion += '.help' + + # Building app + subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = appversion, appname = self.__appname__), shell=True) + + print("Built AppImage version {version}".format(version = appversion)) + + # Cleanup phase, before new run. + for deb in glob.glob(self.appnamedir + '/*.deb'): + os.remove(deb) + subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True) + + def checksums(self): + """Create checksums of the built versions.""" + os.chdir(self.appnamedir) + for appimage in glob.glob('*.AppImage'): + # See if a checksum already exist + if not os.path.exists(appimage + '.md5'): + subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) + + + def move(self, outdir): + """Moves built versions to definitive storage.""" + os.chdir(self.appnamedir) + subprocess.run("find . -iname '*.AppImage*' -exec cp {} %s \;" % outdir, shell=True) + + def __del__(self): + """Destructor""" + # Cleaning up build directory + shutil.rmtree(self.__builddir__) + + +if __name__ == '__main__': + # Run if it is run as a program. + # 1 -> query + # 2 -> arch + # 3 -> url + # 4 -> outdir + + if not len(sys.argv) == 5: + print("Please launch with this parameters: build.py query arch url outputdir") + sys.exit(255) + + b = Build(sys.argv[1], sys.argv[2], sys.argv[3]) + b.download() + b.build() + b.checksums() + b.move(sys.argv[4]) + del b diff --git a/check_updates.sh b/check_updates.sh deleted file mode 100755 index fd557b5..0000000 --- a/check_updates.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -pushd $(dirname $0) -# Checking if pull is successfull -git fetch origin -LC_ALL=C git status | grep -q "Your branch is up to date"; retval=$? - -if [[ ${retval} -ne 0 ]]; then - # Let us update the pip installation - git pull - # for the sake of consistency, let's make the check_updates.sh script - # executable - chmod +x check_updates.sh - pip3 uninstall -y loaih - # build the actual toolkit - python3 setup.py bdist_wheel - pip3 install dist/*.whl; rv=$? - - if [[ ${rv} -eq 0 ]]; then - # cleanup - rm -rf dist build loaih.egg-info - fi -fi -popd diff --git a/fresh.yml b/fresh.yml deleted file mode 100644 index 1f8735a..0000000 --- a/fresh.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -data: - storage: /srv/http/appimage.sys42.eu - download: /var/tmp/downloads - -builds: - - query: fresh - language: basic - offline_help: no - portable: no - - - query: fresh - language: basic - offline_help: yes - portable: no - - - query: fresh - language: basic - offline_help: no - portable: yes - - - query: fresh - language: basic - offline_help: yes - portable: yes - - - query: fresh - language: standard - offline_help: no - portable: no - - - query: fresh - language: standard - offline_help: yes - portable: no - - - query: fresh - language: standard - offline_help: no - portable: yes - - - query: fresh - language: standard - offline_help: yes - portable: yes - - - query: fresh - language: full - offline_help: no - portable: no - - - query: fresh - language: full - offline_help: yes - portable: no - - - query: fresh - language: full - offline_help: no - portable: yes - - - query: fresh - language: full - offline_help: yes - portable: yes diff --git a/getversions.py b/getversions.py new file mode 100644 index 0000000..b6acdd8 --- /dev/null +++ b/getversions.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import urllib.request +from lxml import etree +from packaging.version import parse as parse_version +import re, sys, json + +ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" +RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" +DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/current/" + +def getlatestrel(basever): + """Search in downloadarchive for the latest version matching baseversion.""" + versionlist = etree.HTML(urllib.request.urlopen(ARCHIVE).read()).xpath('//td/a') + # Getting a more polished matching list + cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) + + # Sorting, then returning the last version + return sorted(cleanlist)[-1] + +def getbranchrel(branch): + """Based on branch names, get the release number.""" + basedirurl = {} + version = '' + if branch == 'daily': + basedirurl = { 'x86_64': DAILY, 'x86': '-' } + version = etree.HTML(urllib.request.urlopen(DAILY).read()).xpath('//td/a')[1].text.split('_')[1] + + return { 'version': version, 'basedirurl': basedirurl } + + versions = etree.HTML(urllib.request.urlopen(RELEASE).read()).xpath('//td/a') + index = 1 + if branch == 'still': + index = 2 + elif branch == 'fresh': + index = 3 + version = getlatestrel(versions[index].text.strip('/')) + + return { 'version': version, 'basedirurl': getbaseurl(version) } + +def getbaseurl(version): + """Returns the links based on the numeric version.""" + basediriurl = {} + url = ARCHIVE + '/' + version + '/deb/' + # x86 binaries are not anymore offered after 6.3.0. + if parse_version(version) < parse_version('6.3.0'): + basedirurl['x86'] = url + 'x86/' + else: + basedirurl['x86'] = '-' + + basedirurl['x86_64'] = url + 'x86_64/' + + return basedirurl + +if __name__ == '__main__': + # Preparing variables for outputting + version = '' + basedirurl = {} + basever = 'fresh' + + # At the end of the checks, we need a version string and a basedirurl, which + # should be a dictionaly for x86, x86_64 with the base of the directory where + # to download the files. + if len(sys.argv) > 1: + # A version has been specified. + basever = sys.argv[1] + + # Once overridden with Argv, parse the inputs + if '.' in basever: + # Numbered version. Let's check it is a 4 dotted release + if len(basever.split('.')) == 4: + version = basever + else: + version = getlatestrel(basever) + + basedirurl = getbaseurl(version) + else: + # String versions. + a = getbranchrel(basever) + version = a['version'] + basedirurl = a['basedirurl'] + + output = """RUNDECK:DATA: query = %s +RUNDECK:DATA: version = %s +RUNDECK:DATA: x86 = %s +RUNDECK:DATA: x86_64 = %s""" % (basever, version, basedirurl['x86'] or '-', basedirurl['x86_64']) +print(output) diff --git a/loaih/__init__.py b/loaih/__init__.py deleted file mode 100644 index 89f05d1..0000000 --- a/loaih/__init__.py +++ /dev/null @@ -1,336 +0,0 @@ -#!/usr/bin/env python3 - -import urllib.request -import loaih.versions as versions -from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re - -class Build(object): - LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] - LANGBASIC = [ 'en-GB' ] - - def __init__(self, query, arch): - """Build all versions that can be found in the indicated repo.""" - self.query = query - self.queried_name = False if '.' in self.query else True - self.arch = arch - self.url = {} - self.language = 'basic' - self.offline_help = False - self.portable = False - self.updatable = True - self.storage_path = '/srv/http/appimage.sys42.eu' - self.download_path = '/var/tmp/downloads' - self.appversion = '' - self.appimagefilename = {} - self.genappversion = '' - self.genappimagefilename = {} - - # Getting versions and so on - v = versions.BuildVersion(self.query) - - # Creating a tempfile - self.builddir = tempfile.mkdtemp() - self.tarballs = {} - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' - self.version = v.version - self.url = v.basedirurl - self.built = { u'x86': False, u'x86_64': False } - - # Preparing the default for the relative path on the storage for - # different versions. - # The path will evaluated as part of the check() function, as it is - # understood the storage_path can be changed before that phase. - self.relative_path = [] - self.full_path = '' - - def check(self, storage_path): - """Checking if the requested AppImage has been already built.""" - # Mandate to the private function to calculate the full_path available - # for the storage and the checks. - self.__calculate_full_path__() - - # Incompatibilities - if portable and updatable are asked together, - # only portable will be built. - if self.portable and self.updatable: - print("Upgradable and portable options were required together. Building only portable.") - self.updatable = False - - if self.updatable and not self.queried_name: - # If the queried version was a numbered version, doesn't make sense - # to build an updatable version. - self.updatable = False - - # Building expected AppImageName - self.languagepart = "." - if ',' in self.language: - self.languagepart += self.language.replace(',', '-') - else: - self.languagepart += self.language - - self.helppart = '.help' if self.offline_help else '' - - # If the build was called by queried name, build from latest release available but build with the most generic name - myver = str.join('.', self.version.split('.')[0:2]) - self.genappversion = myver + self.languagepart + self.helppart - self.genappimagefilename[u'x86'] = self.appname + '-' + self.genappversion + '-x86.AppImage' - self.genappimagefilename[u'x86_64'] = self.appname + '-' + self.genappversion + '-x86_64.AppImage' - - self.appversion = self.version + self.languagepart + self.helppart - self.appimagefilename[u'x86'] = self.appname + '-' + self.appversion + '-x86.AppImage' - self.appimagefilename[u'x86_64'] = self.appname + '-' + self.appversion + '-x86_64.AppImage' - - for arch in self.arch: - # For generalized builds, we need to check if there are .ver file - # and it contains the specific version found. - print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) - res = subprocess.run("find {path} -name {appimage}'".format( - path = self.full_path, - appimage = self.genappimagefilename[arch] + '.ver' - ), shell=True, capture_output=True, env={ "LC_ALL": "C" }) - - if "No such file or directory" in res.stderr.decode('utf-8'): - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue - - if res.stdout: - # All good, the command was executed fine. - for file in res.stdout.decode('utf-8').strip('\n').split('\n'): - if self.version in open(file, 'r').read(): - self.built[arch] = True - - print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) - res = subprocess.run("find {path} -name '{appimage}'".format( - path = self.full_path, - appimage = self.appimagefilename[arch] - ), shell=True, capture_output=True) - if res.stdout: - if len(res.stdout.decode('utf-8').strip('\n')) > 1: - self.built[arch] = True - - if self.built[arch]: - print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) - - def __calculate_full_path__(self): - """Calculate relative path of the build, based on internal other variables.""" - if len(self.relative_path) == 0: - if self.query == 'daily': - self.relative_path.append('daily') - elif self.query == 'prerelease': - self.relative_path.append('prerelease') - - # Not the same check, an additional one - if self.portable: - self.relative_path.append('portable') - - fullpath_arr = self.storage_path.split('/') - # Joining relative path only if it is not null - if len(self.relative_path) > 0: - fullpath_arr.extend(self.relative_path) - self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) - - def download(self, download_path): - """Downloads the contents of the URL as it was a folder.""" - # Let's start with defining which files are to be downloaded. - # Let's explore the remote folder. - self.download_path = download_path - - for arch in self.arch: - # Checking if a valid path has been provided - if self.url[arch] == '-': - print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) - # Faking already built it so to skip other checks. - self.built[arch] = True - continue - - if self.built[arch]: - print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) - continue - - contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] - tarballs = self.tarballs[arch] - maintarball = tarballs[0] - - os.makedirs(self.download_path, exist_ok = True) - os.chdir(self.download_path) - for archive in tarballs: - # If the archive is already there, do not do anything. - if os.path.exists(os.path.join(self.download_path, archive)): - print("Archive %s is already there! Sweet" % archive) - continue - - # Download the archive - try: - urllib.request.urlretrieve(self.url[arch] + archive, archive) - except: - print("Failed to download {archive}.".format(archive = archive)) - - print("Got %s." % archive) - - def build(self): - """Building all the versions.""" - # We have 4 builds to do: - # * standard languages, no help - # * standard languages + offline help - # * all languages, no help - # * all languages + offline help - - for arch in self.arch: - if self.built[arch]: - # Already built for arch or path not available. User has already been warned. - continue - - # Preparation tasks - self.appnamedir = os.path.join(self.builddir, self.appname) - self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') - os.makedirs(self.appimagedir, exist_ok = True) - # And then cd to the appname folder. - os.chdir(self.appnamedir) - # Download appimagetool from github - appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) - urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') - os.chmod('appimagetool', 0o755) - - # Build the requested version. - if self.queried_name and not self.portable: - self.__unpackbuild__(arch, True) - self.__unpackbuild__(arch) - - def __unpackbuild__(self, arch, generalize = False): - if generalize and self.portable: - # Doesn't particularly make sense to build a generic portable - # version. Just skipping the specific generic build - return - - # We start by filtering out tarballs from the list - buildtarballs = [ self.tarballs[arch][0] ] - - # Let's process standard languages and append results to the - # buildtarball - if self.language == 'basic': - if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'pack_en-GB' in x ]) - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) - elif self.language == 'standard': - for lang in Build.LANGSTD: - if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack_' + lang) in x ]) - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack_' + lang) in x ]) - elif self.language == 'full': - if self.offline_help: - # We need also all help. Let's replace buildtarball with the - # whole bunch - buildtarballs = self.tarballs[arch] - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) - else: - # Looping for each language in self.language - for lang in self.language.split(","): - if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) - else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - - # Unpacking the tarballs - for archive in buildtarballs: - subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) - - os.chdir(self.appnamedir) - - os.makedirs(self.appimagedir, exist_ok = True) - # At this point, let's decompress the deb packages - subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) - - if self.portable: - shortversion = str.join('.', self.version.split('.')[:3]) - subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) - - # Changing desktop file - subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) - subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) - - subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) - - # Find the name of the binary called in the desktop file. - binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') - - bindir=os.path.join(self.appimagedir, 'usr', 'bin') - os.makedirs(bindir, exist_ok = True) - subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) - - # Download AppRun from github - apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) - dest = os.path.join(self.appimagedir, 'AppRun') - urllib.request.urlretrieve(apprunurl, dest) - os.chmod(dest, 0o755) - - # Building app - if self.updatable: - # Updatable make sense only for generic images for fresh, still, - # daily. If a request was for a specific version, I'd not build an - # updatable version. - # zsync name was generated already - - # If asked to do a generalized build: - if generalize: - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) - # Build version file management - with open(self.genappimagefilename[arch] + '.ver', 'w') as v: - v.write(self.version) - else: - subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) - - else: - if generalize: - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.genappversion, appname = self.appname), shell=True) - with open(self.genappimagefilename[arch] + '.ver', 'w') as v: - v.write(self.version) - else: - subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) - - print("Built AppImage version {version}".format(version = self.appversion)) - - # Cleanup phase, before new run. - for deb in glob.glob(self.appnamedir + '/*.deb'): - os.remove(deb) - subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) - - def checksums(self): - """Create checksums of the built versions.""" - if all(self.built.values()): - # All checksums are already created. - return - - # On the contrary, checksums will be in any case overwritten if - # existent, but generated only for built packages anyways - - os.chdir(self.appnamedir) - for appimage in glob.glob('*.AppImage*'): - if appimage.endswith('.ver'): - # Skipping checksums for .ver files. - continue - - # See if a checksum already exist - if not os.path.exists(appimage + '.md5'): - subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) - - - def publish(self): - """Moves built versions to definitive storage.""" - if all(self.built.values()): - # All files are already present in the full_path - return - - os.chdir(self.appnamedir) - # Forcing creation of subfolders, in case there is a new build - os.makedirs(self.full_path, exist_ok = True) - subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) - - def __del__(self): - """Destructor""" - # Cleaning up build directory - shutil.rmtree(self.builddir) diff --git a/loaih/versions.py b/loaih/versions.py deleted file mode 100644 index fa0d972..0000000 --- a/loaih/versions.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import urllib.request -from lxml import etree -from packaging.version import parse as parse_version - -class BuildVersion(object): - ARCHIVE = "https://downloadarchive.documentfoundation.org/libreoffice/old/" - RELEASE = "https://download.documentfoundation.org/libreoffice/stable/" - DAILY = "https://dev-builds.libreoffice.org/daily/master/Linux-rpm_deb-x86_64@tb87-TDF/" - PRERELEASE = "https://dev-builds.libreoffice.org/pre-releases/deb/x86_64/" - - def __init__(self, query): - self.query = query - self.version = '' - self.basedirurl = {} - - # Parsing the query input. - if '.' in self.query: - # Numbered self.version. Let's check it is a 4 dotted release - if len(self.query.split('.')) == 4: - self.version = self.query - else: - # If not 4 dotted, let's search for the 4 dotted version - self.version = self.__getlatestrel(self.query) - - self.basedirurl = self.__getbaseurl(self.version) - else: - # String self.versions. - a = self.__getbranchrel(self.query) - self.version = a['version'] - self.basedirurl = a['basedirurl'] - - def __getlatestrel(self, basever): - """Search in downloadarchive for the latest version matching baseversion.""" - versionlist = etree.HTML(urllib.request.urlopen(BuildVersion.ARCHIVE).read()).xpath('//td/a') - # Getting a more polished matching list - cleanlist = list(dict.fromkeys([x.text.strip('/') for x in versionlist if x.text.startswith(basever)])) - - # Sorting, then returning the last version - return sorted(cleanlist)[-1] - - def __getbranchrel(self, branch): - """Based on branch names, get the release number.""" - basedirurl = {} - version = '' - if branch == 'daily': - # The daily builds can be mostly distinguished by the day of build - # (official version is constant. - - # The last built version is the next-to-last version [-2] on the page. - fulldailypath = etree.HTML(urllib.request.urlopen(BuildVersion.DAILY).read()).xpath('//td/a')[-2].text - dailyversion = fulldailypath.split('_')[0].replace('-', '') - version - newurl = str.join('/', [ BuildVersion.DAILY, fulldailypath, '' ]) - - basedirurl = { u'x86_64': newurl, u'x86': '-' } - version = etree.HTML(urllib.request.urlopen(newurl).read()).xpath('//td/a')[1].text.split('_')[1] - - return { 'version': version + '-' + dailyversion, 'basedirurl': basedirurl } - - if branch == 'prerelease': - version = etree.HTML(urllib.request.urlopen(BuildVersion.PRERELEASE).read()).xpath('//td/a')[1].text.split('_')[1] - basedirurl = { u'x86': '-', u'x86_64': BuildVersion.PRERELEASE } - - return { 'version': version, 'basedirurl': basedirurl } - - # Stable releases. - versions = etree.HTML(urllib.request.urlopen(BuildVersion.RELEASE).read()).xpath('//td/a') - index = 1 - if branch == 'still': - index = -2 - elif branch == 'fresh': - index = -1 - version = self.__getlatestrel(versions[index].text.strip('/')) - - return { 'version': version, 'basedirurl': self.__getbaseurl(version) } - - def __getbaseurl(self, version): - """Returns the links based on the numeric version.""" - basedirurl = {} - url = BuildVersion.ARCHIVE + '/' + version + '/deb/' - - # x86 binaries are not anymore offered after 6.3.0. - if parse_version(version) < parse_version('6.3.0'): - basedirurl[u'x86'] = url + 'x86/' - else: - basedirurl[u'x86'] = '-' - - basedirurl[u'x86_64'] = url + 'x86_64/' - - return basedirurl diff --git a/scripts/loaih-build b/scripts/loaih-build deleted file mode 100644 index 5ff89a2..0000000 --- a/scripts/loaih-build +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import click -import yaml -import loaih - -@click.command() -@click.option('-a', '--arch', 'arch', type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), default='all', help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") -@click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") -@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") -@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") -@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") -@click.option('-d', '--download-path', 'download', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") -@click.option('-s', '--storage-path', 'storage', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") -@click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") -@click.argument('query') -def build(arch, language, offline, portable, updatable, download, storage, check, query): - # Parsing options - arches = [] - if arch.lower() == 'all': - # We need to build it twice. - arches = [ u'x86', u'x86_64' ] - else: - arches = [ arch.lower().decode('utf-8') ] - - if query.endswith('.yml') or query.endswith('.yaml'): - # This is a buildfile. So we have to load the file and pass the build options ourselves. - config = {} - with open(query, 'r') as file: - config = yaml.safe_load(file) - - # With the config file, we ignore all the command line options and set - # generic default. - for build in config['builds']: - # Loop a run for each build. - obj = loaih.Build(build['query'], arches) - - obj.language = build['language'] - obj.offline_help = build['offline_help'] - obj.portable = build['portable'] - obj.updatable = True - - if check: - obj.check(config['data']['storage']) - else: - obj.storage_path = config['data']['storage'] - - obj.download(config['data']['download']) - obj.build() - obj.checksums() - obj.publish() - del obj - - else: - obj = loaih.Build(query, arches) - - obj.language = language - obj.offline_help = offline - obj.portable = portable - obj.updatable = updatable - - if check: - obj.check(storage) - else: - obj.storage_path = storage - - obj.download(download) - obj.build() - obj.checksums() - obj.publish() - del obj - -if __name__ == '__main__': - build() diff --git a/scripts/loaih-getversion b/scripts/loaih-getversion deleted file mode 100644 index 59d03ae..0000000 --- a/scripts/loaih-getversion +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import click -from loaih.versions import BuildVersion -import re, sys, json - -@click.command() -@click.option('-o', '--output', default = 'rundeck', type=click.Choice(['rundeck', 'json', 'text' ], case_sensitive=False), help="Output format, defaulting to Rundeck Key/Value data format. Options: rundeck,json,text") -@click.argument('query') -def getversion(query, output): - b = BuildVersion(query) - - if output.lower() == 'rundeck': - print("""RUNDECK:DATA: query = {query} -RUNDECK:DATA: version = {version} -RUNDECK:DATA: x86 = {x86_url} -RUNDECK:DATA: x86_64 = {x86_64_url}""".format(query = query, version = b.version, x86_url = b.basedirurl['x86'], x86_64_url = b.basedirurl['x86_64'])) - elif output.lower() == 'json': - output = { - 'query': query, - 'version': b.version, - 'basedirurl': b.basedirurl - } - print(json.dumps(output)) - else: - print("""query: {query} -version: {version} -x86: {x86_url} -x86_64: {x86_64_url}""".format(query = query, version = b.version, x86_url = b.basedirurl['x86'], x86_64_url = b.basedirurl['x86_64'])) - - -if __name__ == '__main__': - getversion() diff --git a/setup.py b/setup.py deleted file mode 100644 index 032157f..0000000 --- a/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -# vim:sts=4:sw=4 - -from setuptools import setup,find_packages - -setup( - name="loaih", - version="1.0.1", - description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", - author="Emiliano Vavassori", - author_email="syntaxerrormmm@libreoffice.org", - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - scripts=[ 'scripts/loaih-getversion', 'scripts/loaih-build' ], - install_requires=[ 'click', ], - license='MIT', - url='https://git.libreitalia.org/LibreItalia/loappimage-helpers/', -) diff --git a/still.yml b/still.yml deleted file mode 100644 index 3d86642..0000000 --- a/still.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -data: - storage: /srv/http/appimage.sys42.eu - download: /var/tmp/downloads - -builds: - - query: still - language: basic - offline_help: no - portable: no - - - query: still - language: basic - offline_help: yes - portable: no - - - query: still - language: basic - offline_help: no - portable: yes - - - query: still - language: basic - offline_help: yes - portable: yes - - - query: still - language: standard - offline_help: no - portable: no - - - query: still - language: standard - offline_help: yes - portable: no - - - query: still - language: standard - offline_help: no - portable: yes - - - query: still - language: standard - offline_help: yes - portable: yes - - - query: still - language: full - offline_help: no - portable: no - - - query: still - language: full - offline_help: yes - portable: no - - - query: still - language: full - offline_help: no - portable: yes - - - query: still - language: full - offline_help: yes - portable: yes