Prima implementazione controllo build remota e caricamento con rsync + ssh.

loaih/build.py

@@ -32,6 +32,9 @@ class Build(loaih.RemoteBuild):
         self.portable = False
         self.updatable = True
         self.sign = True
+        self.remoterepo = False
+        self.remote_host = ''
+        self.remote_path = ''
         self.storage_path = '/mnt/appimage'
         self.download_path = '/var/tmp/downloads'
 
@@ -90,7 +93,7 @@ class Build(loaih.RemoteBuild):
             if self.query == 'daily':
                 self.relative_path.append('daily')
             elif self.query == 'prerelease':
-                self.relative_path.append('prerelease') 
+                self.relative_path.append('prerelease')
 
             # Not the same check, an additional one
             if self.portable:
@@ -105,25 +108,44 @@ class Build(loaih.RemoteBuild):
 
     def check(self):
         """Checking if the requested AppImage has been already built."""
-        if not len(self.appimagefilename) == 2:
+        if len(self.appimagefilename) != 2:
             self.calculate()
 
         for arch in self.arch:
-            print(f"Searching for {self.appimagefilename[arch]}")
-            res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8')
 
-            if "No such file or directory" in res.stderr:
+            # First, check if by metadata the repo is remote or not.
-                # Folder is not existent: so the version was not built
+            if self.remoterepo or 'http' in self.storage_path:
-                # Build stays false, and we go to the next arch
+                self.remoterepo = True
-                continue
+                # Remote storage. I have to query a remote site to know if it
+                # was already built.
+                name = self.appimagefilename[arch]
+                matching = etree.HTML(urllib.request.urlopen(str.join('/',
+                    self.relative_path.insert(0, self.storage_path)
+                )).read()).xpath(
+                    f"//a[contains(@href, '{name}')/@href"
+                )
 
-            if res.stdout and len(res.stdout.strip("\n")) > 0:
+                if len(matching) > 0:
-                # All good, the command was executed fine.
+                    # Already built.
-                print(f"Build for {self.version} found.")
+                    self.built[arch] = True
-                self.built[arch] = True
+
+            else:
+                # Repo is local
+                print(f"Searching for {self.appimagefilename[arch]}")
+                res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8')
+
+                if "No such file or directory" in res.stderr:
+                    # Folder is not existent: so the version was not built
+                    # Build stays false, and we go to the next arch
+                    continue
+
+                if res.stdout and len(res.stdout.strip("\n")) > 0:
+                    # All good, the command was executed fine.
+                    print(f"Build for {self.version} found.")
+                    self.built[arch] = True
 
             if self.built[arch]:
-                print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.")
+                    print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.")
 
 
     def download(self):
@@ -309,13 +331,33 @@ class Build(loaih.RemoteBuild):
             return
 
         os.chdir(self.appnamedir)
-        # Forcing creation of subfolders, in case there is a new build
+
-        os.makedirs(self.full_path, exist_ok = True)
+        # Two cases here: local and remote storage_path.
-        for file in glob.glob("*.AppImage*"):
+        if self.remoterepo:
-            subprocess.run(shlex.split(f"cp -f {file} {self.full_path}"))
+            # Remote first.
+            # Build destination directory
+            if len(self.relative_path) > 0:
+                remotepath = str.join('/', self.relative_path.insert(0, self.remote_path))
+            else:
+                remotepath = str.join('/', [ self.remote_path, '' ])
+            try:
+                subprocess.run(
+                    shlex.split(
+                        f"rsync -avz -e ssh *.AppImage* {self.remote_host}:{remotepath}"
+                    )
+                )
+            finally:
+                pass
+
+        else:
+            # Local
+            # Forcing creation of subfolders, in case there is a new build
+            os.makedirs(self.full_path, exist_ok = True)
+            for file in glob.glob("*.AppImage*"):
+                subprocess.run(shlex.split(f"cp -f {file} {self.full_path}"))
 
 
-    def generalize_and_link(self):
+    def generalize_and_link(self, chdir = self.full_path):
         """Creates the needed generalized files if needed."""
         # If called with a pointed version, no generalize and link necessary.
         if not self.branch_version:
@@ -335,7 +377,7 @@ class Build(loaih.RemoteBuild):
             if self.built[arch]:
                 continue
 
-            os.chdir(self.full_path)
+            os.chdir(chdir)
             # if the appimage for the reported arch is not found, skip to next
             # arch
             if not os.path.exists(self.appimagefilename[arch]):

loaih/script.py

@@ -71,6 +71,10 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
                 obj.updatable = True
                 obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu'
                 obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads'
+                if 'http' in obj.storage_path:
+                    obj.remoterepo = True
+                    obj.remote_host = config['data']['remote_host'] if 'remote_host' in config['data'] and config['data']['remote_host'] else 'ciccio.libreitalia.org'
+                    obj.remote_path = config['data']['remote_path'] if 'remote_path' in config['data'] and config['data']['remote_path'] else '/var/lib/nethserver/vhost/appimages'
 
                 if 'sign' in config['data'] and config['data']['sign']:
                     obj.sign = True
@@ -83,8 +87,11 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
                 obj.download()
                 obj.build()
                 obj.checksums()
+                if obj.remoterepo:
+                    obj.generalize_and_link(obj.appnamedir)
                 obj.publish()
-                obj.generalize_and_link()
+                if not obj.remoterepo:
+                    obj.generalize_and_link()
                 del obj
 
     else: