nouveau système de CI utilisant les webhooks

overlays/site-max.nix

@@ -1,5 +0,0 @@
-self: super:
-
-{
-  site-max = super.callPackage ../pkgs/site-max { };
-}

services/python-ci.nix

@@ -0,0 +1,41 @@
+{lib, config, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.python-ci;
+in
+{
+  options.services.python-ci = {
+    enable = mkEnableOption "Service de CI Nix écrit en Python";
+  };
+
+  config = mkIf cfg.enable {
+
+    systemd.services.python-ci = {
+      description = "CI Nix en Python";
+      requires = ["network-online.target"];
+      wantedBy = ["multi-user.target"];
+      environment = { HOME = "/var/lib/python-ci"; NIX_PATH = concatStringsSep ":" config.nix.nixPath; NIXPKGS_ALLOW_UNFREE = "1";};
+      path = with pkgs;[ nix gnutar gzip ];
+      serviceConfig = {
+        DynamicUser = true;
+        StateDirectory = "python-ci";
+        RuntimeDirectory = "python-ci";
+        RuntimeDirectoryPreserve = "yes";
+        ExecStart = with pkgs;
+        let env = python3Packages.python.buildEnv.override {
+          extraLibs = with python3Packages;[ pyramid python-gitlab ];
+          ignoreCollisions = true;
+        };
+        in "${pkgs.writeShellScriptBin "run.sh" ''
+          ${env}/bin/python ${pkgs.writeScript "python-ci.py" "${readFile ./python-ci.py}"} --port 52350 \
+              --secret /var/lib/python-ci/secret --gitlab-token /var/lib/python-ci/gitlab_token \
+              --gitea-token /var/lib/python-ci/gitea_token --output /run/python-ci
+        ''}/bin/run.sh";
+      };
+    };
+
+  };
+
+}

services/python-ci.py

@@ -0,0 +1,153 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p "python3.withPackages(ps: [ps.pyramid ps.python-gitlab])"
+from wsgiref.simple_server import make_server
+from pyramid.config import Configurator
+from pyramid.view import view_config, view_defaults
+from pyramid.httpexceptions import HTTPNotFound
+from subprocess import check_call, CalledProcessError
+import urllib.request
+import tarfile
+from tempfile import TemporaryDirectory
+from multiprocessing import Pool
+from gitlab import Gitlab
+import urllib.request
+import json
+import argparse
+
+
+def gitlab_build(payload, gl):
+    commit = gl.projects.get(payload['project']['path_with_namespace']).commits.get(payload['checkout_sha'])
+
+    commit.statuses.create({'state': 'running', 'name': 'Python CI'})
+    print("push from " + payload['user_name'])
+    print("repo: " + payload['project']['path_with_namespace'])
+    print("commit: " + payload['checkout_sha'])
+    temp_dir = TemporaryDirectory()
+    repo_dir = temp_dir.name + '/' + payload['project']['name'] + '-' + payload['checkout_sha']
+    archive_url = payload['project']['web_url'] + '/-/archive/' + payload['checkout_sha'] + \
+                  '/' + payload['project']['name'] + '-' + payload['checkout_sha'] + '.tar.gz'
+
+    with urllib.request.urlopen(archive_url) as gitlab_archive:
+        with tarfile.open(fileobj=gitlab_archive, mode='r|gz') as gitlab_repo_files:
+            gitlab_repo_files.extractall(path=temp_dir.name)
+
+    check_call(['ls', '-lha', repo_dir])
+
+    try:
+        check_call(['nix-build', '-o', args.output + '/' + payload['project']['path_with_namespace'], repo_dir])
+    except CalledProcessError:
+        commit.statuses.create({'state': 'failed', 'name': 'Python CI'})
+        print("erreur build")
+    else:
+        commit.statuses.create({'state': 'success', 'name': 'Python CI'})
+        print("build terminé")
+
+
+@view_defaults(
+        route_name="gitlab_payload", renderer="json", request_method="POST"
+)
+class GitlabHook(object):
+
+    def __init__(self, request):
+        self.request = request
+        self.payload = self.request.json
+        self.whitelist = ['nyanloutre/site-musique']
+        self.secret = open(args.secret, 'r').readline().splitlines()[0]
+        self.gitlab_token = open(args.gitlab_token, 'r').readline().splitlines()[0]
+        self.gl = Gitlab('https://gitlab.com', private_token=self.gitlab_token)
+
+    @view_config(header="X-Gitlab-Event:Push Hook")
+    def push_hook(self):
+        if self.payload['project']['path_with_namespace'] in self.whitelist and self.request.headers['X-Gitlab-Token'] == self.secret:
+            self.gl.projects.get(self.payload['project']['path_with_namespace']).commits.get(self.payload['checkout_sha']).statuses.create({'state': 'pending', 'name': 'Python CI'})
+            pool.apply_async(gitlab_build, (self.payload, self.gl))
+            return "build started"
+        else:
+            raise HTTPNotFound
+
+
+def gitea_status_update(repo, commit, token, status):
+    url = 'https://gitea.nyanlout.re/api/v1/repos/' + repo + '/statuses/' + commit
+    print(url)
+    req = urllib.request.Request(url)
+    req.add_header('Content-Type', 'application/json; charset=utf-8')
+    req.add_header('accept', 'application/json')
+    req.add_header('Authorization', 'token ' + token)
+
+    jsondata = json.dumps({'state': status}).encode('utf-8')
+    req.add_header('Content-Length', len(jsondata))
+
+    urllib.request.urlopen(req, jsondata)
+
+def gitea_build(payload, token):
+    commit = payload['after']
+    repo = payload['repository']['full_name']
+
+    gitea_status_update(repo, commit, token, 'pending')
+
+    print("push from " + payload['pusher']['username'])
+    print("repo: " + repo)
+    print("commit: " + commit)
+    temp_dir = TemporaryDirectory()
+    repo_dir = temp_dir.name + '/' + payload['repository']['name']
+    archive_url = payload['repository']['html_url'] + '/archive/' + commit + '.tar.gz'
+
+    with urllib.request.urlopen(archive_url) as gitea_archive:
+        with tarfile.open(fileobj=gitea_archive, mode='r|gz') as gitea_repo_files:
+            gitea_repo_files.extractall(path=temp_dir.name)
+
+    check_call(['ls', '-lha', repo_dir])
+
+    try:
+        check_call(['nix-build', '-o', args.output + '/' + repo, repo_dir])
+    except CalledProcessError:
+        gitea_status_update(repo, commit, token, 'failure')
+        print("erreur build")
+    else:
+        gitea_status_update(repo, commit, token, 'success')
+        print("build terminé")
+
+
+@view_defaults(
+        route_name="gitea_payload", renderer="json", request_method="POST"
+)
+class GiteaHook(object):
+
+    def __init__(self, request):
+        self.request = request
+        self.payload = self.request.json
+        self.whitelist = ['nyanloutre/site-musique', 'nyanloutre/site-max']
+        self.secret = open(args.secret, 'r').readline().splitlines()[0]
+        self.gitea_token = open(args.gitea_token, 'r').readline().splitlines()[0]
+
+    @view_config(header="X-Gitea-Event:push")
+    def push_hook(self):
+        if self.payload['repository']['full_name'] in self.whitelist and self.payload['secret'] == self.secret:
+            pool.apply_async(gitea_build, (self.payload, self.gitea_token))
+            return "build started"
+        else:
+            raise HTTPNotFound
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='CI server')
+    parser.add_argument('--address', help='listening address', default='127.0.0.1')
+    parser.add_argument('--port', type=int, help='listening port')
+    parser.add_argument('--output', help='output directory')
+    parser.add_argument('--secret', help='repo secret file')
+    parser.add_argument('--gitlab-token', help='gitlab token file')
+    parser.add_argument('--gitea-token', help='gitea token file')
+    args = parser.parse_args()
+
+    pool = Pool(1)
+
+    config = Configurator()
+
+    config.add_route("gitlab_payload", "/gitlab_payload")
+    config.add_route("gitea_payload", "/gitea_payload")
+    config.scan()
+
+    app = config.make_wsgi_app()
+    server = make_server(args.address, args.port, app)
+    print('listening ...')
+    server.serve_forever()

services/site-max.nix

@@ -24,10 +24,6 @@ in
 
   config = mkIf cfg.enable {
 
-    nixpkgs.overlays = [
-      (import ../overlays/site-max.nix)
-    ];
-
     services.haproxy-acme.services = {
       ${cfg.domaine} = { ip = "127.0.0.1"; port = cfg.port; auth = false; };
     };
@@ -37,46 +33,10 @@ in
         "max" = {
           listen = [ { addr = "127.0.0.1"; port = cfg.port; } ];
           locations."/" = {
-            root = "/run/site-max/result";
+            root = "/run/python-ci/nyanloutre/site-max";
           };
         };
       };
     };
-
-    systemd.services.build-site-max = {
-      description = "Compilation du site de Max Spiegel";
-      requires = ["network-online.target"];
-      path = with pkgs;[ git nix ];
-      environment = { HOME = "/var/lib/site-max"; NIX_PATH = "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs"; };
-
-      serviceConfig = {
-        DynamicUser = true;
-        RuntimeDirectory = "site-max";
-        RuntimeDirectoryPreserve = "yes";
-        CacheDirectory = "site-max";
-        Type = "oneshot";
-        ExecStart = "${pkgs.writeShellScriptBin "build.sh" ''
-          set -x
-          set -e
-          GIT_CLONE_DIR=/var/cache/site-max
-
-          if [ ! -d $GIT_CLONE_DIR/.git ]; then
-            git clone --depth 1 https://github.com/nyanloutre/site-max.git $GIT_CLONE_DIR
-          else
-            git -C $GIT_CLONE_DIR pull
-          fi
-
-          NIXPKGS_ALLOW_UNFREE=1 nix-build -o /run/site-max/result $GIT_CLONE_DIR
-        ''}/bin/build.sh";
-      };
-    };
-
-    systemd.timers.build-site-max = {
-      description = "Timer de compilation du site de Max";
-      requires = ["network-online.target"];
-      wantedBy = ["multi-user.target"];
-      timerConfig = { OnCalendar = "*:0/5"; Unit = "build-site-max.service"; };
-    };
-
   };
 }

services/site-musique.nix

@@ -32,7 +32,7 @@ in
       "musique" = {
         listen = [ { addr = "127.0.0.1"; port = cfg.port; } ];
         locations."/" = {
-          root = "/run/site-musique/result";
+          root = "/run/python-ci/nyanloutre/site-musique";
           index = "index.php";
           extraConfig = ''
             location ~* \.php$ {
@@ -62,40 +62,5 @@ in
       php_admin_flag[log_errors] = on
       catch_workers_output = yes
     '';
-
-    systemd.services.build-site-musique = {
-      description = "Compilation du site de la musique";
-      requires = ["network-online.target"];
-      path = with pkgs;[ git nix ];
-      environment = { HOME = "/var/lib/site-musique"; NIX_PATH = "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos/nixpkgs"; };
-
-      serviceConfig = {
-        DynamicUser = true;
-        RuntimeDirectory = "site-musique";
-        RuntimeDirectoryPreserve = "yes";
-        CacheDirectory = "site-musique";
-        Type = "oneshot";
-        ExecStart = "${pkgs.writeShellScriptBin "build.sh" ''
-          set -x
-          set -e
-          GIT_CLONE_DIR=/var/cache/site-musique
-
-          if [ ! -d $GIT_CLONE_DIR/.git ]; then
-            git clone --depth 1 https://gitlab.com/nyanloutre/site-musique.git $GIT_CLONE_DIR
-          else
-            git -C $GIT_CLONE_DIR pull
-          fi
-
-          NIXPKGS_ALLOW_UNFREE=1 nix-build -o /run/site-musique/result $GIT_CLONE_DIR
-        ''}/bin/build.sh";
-      };
-    };
-
-    systemd.timers.build-site-musique = {
-      description = "Timer de compilation du site de la musique";
-      requires = ["network-online.target"];
-      wantedBy = ["multi-user.target"];
-      timerConfig = { OnCalendar = "*:0/5"; Unit = "build-site-musique.service"; };
-    };
   };
 }

systems/LoutreOS/services.nix

@@ -18,6 +18,7 @@ in
     ../../services/site-musique.nix
     ../../services/site-max.nix
     ../../services/auto-pr.nix
+    ../../services/python-ci.nix
     ../../containers/vsftpd.nix
   ];
 
@@ -56,6 +57,7 @@ in
         "matrix.${domaine}" = { ip = "127.0.0.1"; port = 8008; auth = false; };
         "pgmanage.${domaine}" = { ip = "127.0.0.1"; port = pgmanage_port; auth = true; };
         "gitea.${domaine}" = { ip = "127.0.0.1"; port = 3001; auth = false; };
+        "ci.${domaine}" = { ip = "127.0.0.1"; port = 52350; auth = false; };
       };
     };
 
@@ -356,6 +358,8 @@ in
     };
 
     auto-pr.enable = true;
+
+    python-ci.enable = true;
   };
 
   /*