Fix tentativo per creazione di link corretti e zsync truccati.

loaih/__init__.py

@@ -3,7 +3,7 @@
 import urllib.request
 import loaih.versions as versions
 from lxml import etree
-import tempfile, os, sys, glob, subprocess, shutil, re
+import tempfile, os, sys, glob, subprocess, shutil, re, shlex
 
 class Build(object):
     LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
@@ -13,32 +13,34 @@ class Build(object):
     def __init__(self, query, arch):
         """Build all versions that can be found in the indicated repo."""
         self.query = query
-        self.queried_name = False if '.' in self.query else True
         self.arch = arch
-        self.url = {}
+        
+        # Getting versions and so on
+        v = versions.BuildVersion(self.query) 
+        self.version = v.version
+        self.short_version = str.join('.', self.version.split('.')[0:2])
+        self.branch_version = None
+        if not '.' in self.query:
+            self.branch_version = self.query
+        self.url = v.basedirurl
+
+        # Other default values
         self.language = 'basic'
         self.offline_help = False
         self.portable = False
         self.updatable = True
-        self.sign = False
+        self.sign = True
-        self.storage_path = '/srv/http/appimage.sys42.eu'
+        self.storage_path = '/mnt/appimage'
         self.download_path = '/var/tmp/downloads'
 
         # Specific build version
         self.appversion = ''
-        self.genappversion = ''
         self.appimagefilename = {}
-        self.genappimagefilename = {}
+        self.zsyncfilename = {}
-
-        # Getting versions and so on
-        v = versions.BuildVersion(self.query) 
 
         # Creating a tempfile
         self.builddir = tempfile.mkdtemp()
         self.tarballs = {}
-        self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
-        self.version = v.version
-        self.url = v.basedirurl
         self.built = { u'x86': False, u'x86_64': False }
         
         # Preparing the default for the relative path on the storage for
@@ -51,73 +53,32 @@ class Build(object):
 
     def calculate(self):
         """Calculate exclusions and other variables."""
-        # Incompatibilities - if portable and updatable are asked together,
+        # AppName
-        # only portable will be built.
+        self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
-        if self.portable and self.updatable:
-            print("Upgradable and portable options were required together. Building only portable.")
-            self.updatable = False
 
-        if self.updatable and not self.queried_name:
+        # Calculating languagepart
-            # If the queried version was a numbered version, doesn't make sense
-            # to build an updatable version.
-            self.updatable = False
-
-        # Mandate to the private function to calculate the full_path available
-        # for the storage and the checks.
-        self.__calculate_full_path__()
-
-        # Building expected AppImageName
         self.languagepart = "."
         if ',' in self.language:
             self.languagepart += self.language.replace(',', '-')
         else:
             self.languagepart += self.language
 
+        # Calculating help part
         self.helppart = '.help' if self.offline_help else ''
 
-        # If the build was called by queried name, build from latest release available but build with the most generic name
+        # Building the required names
-        self.appversion = self.version + self.languagepart + self.helppart
-        myver = str.join('.', self.version.split('.')[0:2])
-        self.genappversion = myver + self.languagepart + self.helppart
         for arch in Build.ARCHSTD:
-            self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage'
+            self.appimagefilename[arch] = __genappimagefilename__(self.version, arch)
-            self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage'
+            self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync'
+
+        # Mandate to the private function to calculate the full_path available
+        # for the storage and the checks.
+        self.__calculate_full_path__()
 
 
-    def check(self):
+    def __gen_appimagefilename__(self, version, arch):
-        """Checking if the requested AppImage has been already built."""
+        """Generalize the construction of the name of the app."""
-
+        return self.appname + f'-{version}' + self.languagepart + self.helppart + f'-{arch}.AppImage'
-        for arch in self.arch:
-            # For generalized builds, we need to check if there are .ver file
-            # and it contains the specific version found.
-            print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver'))
-            res = subprocess.run("find {path} -name {appimage}'".format(
-                path = self.full_path,
-                appimage = self.genappimagefilename[arch] + '.ver'
-            ), shell=True, capture_output=True, env={ "LC_ALL": "C" })
-
-            if "No such file or directory" in res.stderr.decode('utf-8'):
-                # Folder is not existent: so the version was not built
-                # Build stays false, and we go to the next arch
-                continue
-
-            if res.stdout:
-                # All good, the command was executed fine.
-                for file in res.stdout.decode('utf-8').strip('\n').split('\n'):
-                    if self.version in open(file, 'r').read():
-                        self.built[arch] = True
-
-            print("Debug: searching for {file}".format(file = self.appimagefilename[arch]))
-            res = subprocess.run("find {path} -name '{appimage}'".format(
-                path = self.full_path,
-                appimage = self.appimagefilename[arch]
-            ), shell=True, capture_output=True)
-            if res.stdout:
-                if len(res.stdout.decode('utf-8').strip('\n')) > 1:
-                    self.built[arch] = True
-
-            if self.built[arch]:
-                print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch))
 
 
     def __calculate_full_path__(self):
@@ -139,49 +100,70 @@ class Build(object):
         self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr))
 
 
+    def check(self):
+        """Checking if the requested AppImage has been already built."""
+        if not len(self.appimagefilename) == 2:
+            self.calculate()
+
+        for arch in self.arch:
+            print(f"Searching for {self.appimagefilename[arch]}")
+            res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}'"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8')
+
+            if "No such file or directory" in res.stderr:
+                # Folder is not existent: so the version was not built
+                # Build stays false, and we go to the next arch
+                continue
+
+            if res.stdout:
+                # All good, the command was executed fine.
+                for file in res.stdout.strip('\n').split('\n'):
+                    if self.version in open(file, 'r').read():
+                        print(f"Build for {self.version} found.")
+                        self.built[arch] = True
+
+            if self.built[arch]:
+                print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.")
+
+
     def download(self):
         """Downloads the contents of the URL as it was a folder."""
+        print(f"Started downloads for {self.version}. Please wait.")
         for arch in self.arch:
             # Checking if a valid path has been provided
             if self.url[arch] == '-':
-                print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch))
+                print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.")
                 # Faking already built it so to skip other checks.
                 self.built[arch] = True
                 continue
 
             if self.built[arch]:
-                print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch))
+                print(f"A build for {arch} was already found. Skipping specific packages.")
                 continue
 
+            # Identifying downloads
             contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a")
             self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ]
             tarballs = self.tarballs[arch]
             maintarball = tarballs[0]
 
+            # Create and change directory to the download location
             os.makedirs(self.download_path, exist_ok = True)
             os.chdir(self.download_path)
             for archive in tarballs:
                 # If the archive is already there, do not do anything.
-                if os.path.exists(os.path.join(self.download_path, archive)):
+                if os.path.exists(archive):
-                    print("Archive %s is already there! Sweet" % archive)
                     continue
 
                 # Download the archive
                 try:
                     urllib.request.urlretrieve(self.url[arch] + archive, archive)
                 except:
-                    print("Failed to download {archive}.".format(archive = archive))
+                    print(f"Failed to download {archive}.")
-
-                print("Got %s." % archive)
 
+        print(f"Finished downloads for {self.version}.")
 
     def build(self):
         """Building all the versions."""
-        # We have 4 builds to do:
-        # * standard languages, no help
-        # * standard languages + offline help
-        # * all languages, no help
-        # * all languages + offline help
 
         for arch in self.arch:
             if self.built[arch]:
@@ -190,28 +172,18 @@ class Build(object):
 
             # Preparation tasks
             self.appnamedir = os.path.join(self.builddir, self.appname)
-            self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
-            os.makedirs(self.appimagedir, exist_ok = True)
             # And then cd to the appname folder.
             os.chdir(self.appnamedir)
             # Download appimagetool from github
-            appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch)
+            appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage"
             urllib.request.urlretrieve(appimagetoolurl, 'appimagetool')
             os.chmod('appimagetool', 0o755)
 
             # Build the requested version.
-            if self.queried_name and not self.portable:
-                # If it is portable, do not generate a generalized version
-                self.__unpackbuild__(arch, True)
             self.__unpackbuild__(arch)
 
 
-    def __unpackbuild__(self, arch, generalize = False):
+    def __unpackbuild__(self, arch):
-        if generalize and self.portable:
-            # Doesn't particularly make sense to build a generic portable
-            # version. Just skipping the specific generic build
-            return
-
         # We start by filtering out tarballs from the list
         buildtarballs = [ self.tarballs[arch][0] ] 
 
@@ -243,35 +215,38 @@ class Build(object):
                 else:
                     buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ])
                 
-        # Unpacking the tarballs
-        for archive in buildtarballs:
-            subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True)
-
         os.chdir(self.appnamedir)
 
+        # Unpacking the tarballs
+        for archive in buildtarballs:
+            subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}"))
+
+        # create appimagedir
+        self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
         os.makedirs(self.appimagedir, exist_ok = True)
+
         # At this point, let's decompress the deb packages
-        subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir)
+        subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir)
 
         if self.portable:
-            shortversion = str.join('.', self.version.split('.')[:3])
+            subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir)
-            subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir)
 
         # Changing desktop file
-        subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir)
+        subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir)
-        subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir)
+        subprocess.run(shlex.split("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname), cwd=self.appimagedir)
 
-        subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir)
+        subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir)
 
         # Find the name of the binary called in the desktop file.
-        binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n')
+        binary_exec = subprocess.run(shlex.split("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
+        binaryname = binary_exec.stdout.strip("\n")
 
         bindir=os.path.join(self.appimagedir, 'usr', 'bin')
         os.makedirs(bindir, exist_ok = True)
-        subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir)
+        subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir)
 
         # Download AppRun from github
-        apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch)
+        apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}"
         dest = os.path.join(self.appimagedir, 'AppRun')
         urllib.request.urlretrieve(apprunurl, dest)
         os.chmod(dest, 0o755)
@@ -280,57 +255,36 @@ class Build(object):
         buildopts = []
         if self.sign:
             buildopts.append('--sign')
-        
+
-        # Building app
+        # adding zsync build if updatable
         if self.updatable:
-            # Updatable make sense only for generic images for fresh, still,
+            buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'")
-            # daily. If a request was for a specific version, I'd not build an
-            # updatable version.
-            # zsync name was generated already
 
-            # If asked to do a generalized build:
+        buildopts_str = str.join(' ', buildopts)
-            if generalize:
+        # Build the number-specific build
-                subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
+        subprocess.run(shlex.split(f"VERSION={self.appversion} ./appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"))
-                # Build version file management
+        
-                with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
+        print(f"Built AppImage version {self.appversion}")
-                    v.write(self.version)
-            else:
-                subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
-
-        else:
-            if generalize:
-                subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
-                with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
-                    v.write(self.version)
-            else:
-                subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
-
-        print("Built AppImage version {version}".format(version = self.appversion))
 
         # Cleanup phase, before new run.
         for deb in glob.glob(self.appnamedir + '/*.deb'):
             os.remove(deb)
-        subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True)
+        subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+"))
 
 
     def checksums(self):
         """Create checksums of the built versions."""
+        # Skip checksum if initally the build was already found in the storage directory
         if all(self.built.values()):
-            # All checksums are already created.
+            return
-            return 
-
-        # On the contrary, checksums will be in any case overwritten if
-        # existent, but generated only for built packages anyways
 
         os.chdir(self.appnamedir)
-        for appimage in glob.glob('*.AppImage*'):
+        for arch in self.arch:
-            if appimage.endswith('.ver'):
+            for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]:
-                # Skipping checksums for .ver files.
+                # For any built arch, find out if a file exist.
-                continue
+                if len(glob.glob(self.appimagefilename[arch] + '.md5')) == 0:
-
+                    # Build checksum
-            # See if a checksum already exist
+                    subprocess.run(shlex.split(f"md5sum {item} > {item}.md5"))
-            if not os.path.exists(appimage + '.md5'):
-                subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True)
 
 
     def publish(self):
@@ -342,7 +296,41 @@ class Build(object):
         os.chdir(self.appnamedir)
         # Forcing creation of subfolders, in case there is a new build
         os.makedirs(self.full_path, exist_ok = True)
-        subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True)
+        for file in glob.glob("*.AppImage*"):
+            subprocess.run(shlex.split(f"cp -f {file} {self.fullpath}"))
+
+
+    def generalize_and_link(self):
+        """Creates the needed generalized files if needed."""
+        # If called with a pointed version, no generalize and link necessary.
+        if not self.branch_version:
+            return
+
+        # Creating versions for short version and query text
+        versions = [ self.short_version, self.branch_version ]
+        for arch in Build.ARCHSTD:
+            # if the appimage for the reported arch is not found, skip to next
+            # arch
+            if not os.path.exists(self.appimagefilename[arch]):
+                continue
+
+            # Doing it both for short_name and for branchname
+            for version in versions:
+                appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage'
+                zsyncfilename[arch] = appimagefilename[arch] + '.zsync'
+
+                os.chdir(self.full_path)
+                # Create the symlink
+                os.symlink(self.appimagefilename[arch], appimagefilename[arch])
+                # Create the checksum for the AppImage
+                subprocess.run(shlex.split("md5sum {item} > {item}.md5".format(item=appimagefilename[arch])))
+                # Do not continue if no zsync file is provided.
+                if not self.updatable:
+                    continue
+
+                os.copy(self.zsyncfilename[arch], zsyncfilename[arch])
+                # Editing the zsyncfile
+                subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}"))
 
 
     def __del__(self):

scripts/loaih-build

@@ -12,7 +12,7 @@ import loaih
 @click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic")
 @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help")
 @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable")
-@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu")
+@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage")
 @click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign")
 @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable")
 @click.argument('query')
@@ -57,6 +57,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
             obj.build()
             obj.checksums()
             obj.publish()
+            obj.generalize_and_link()
             del obj
 
     else: